From 059cc57ba6914229dbd7238b902ba76d87643c6e Mon Sep 17 00:00:00 2001 From: Takeshi Nakatani Date: Sat, 3 Oct 2020 02:14:23 +0000 Subject: [PATCH] Added atime and Corrected atime/mtime/ctime operations --- src/cache.cpp | 12 ++ src/fdcache_entity.cpp | 137 +++++++++++++++--- src/fdcache_entity.h | 12 +- src/metaheader.cpp | 34 +++-- src/metaheader.h | 2 +- src/s3fs.cpp | 112 ++++++++++----- test/integration-test-main.sh | 264 ++++++++++++++++++++++++++++++---- test/test-utils.sh | 8 ++ 8 files changed, 484 insertions(+), 97 deletions(-) diff --git a/src/cache.cpp b/src/cache.cpp index 3e5ee8c..e799061 100644 --- a/src/cache.cpp +++ b/src/cache.cpp @@ -771,9 +771,21 @@ bool convert_header_to_stat(const char* path, const headers_t& meta, struct stat // mtime pst->st_mtime = get_mtime(meta); + if(pst->st_mtime < 0){ + pst->st_mtime = 0L; + } // ctime pst->st_ctime = get_ctime(meta); + if(pst->st_ctime < 0){ + pst->st_ctime = 0L; + } + + // atime + pst->st_atime = get_atime(meta); + if(pst->st_atime < 0){ + pst->st_atime = 0L; + } // size pst->st_size = get_size(meta); diff --git a/src/fdcache_entity.cpp b/src/fdcache_entity.cpp index 263b2df..71c2892 100644 --- a/src/fdcache_entity.cpp +++ b/src/fdcache_entity.cpp @@ -96,7 +96,7 @@ ino_t FdEntity::GetInode(int fd) FdEntity::FdEntity(const char* tpath, const char* cpath) : is_lock_init(false), refcnt(0), path(SAFESTRPTR(tpath)), fd(-1), pfile(NULL), inode(0), size_orgmeta(0), upload_id(""), mp_start(0), mp_size(0), - cachepath(SAFESTRPTR(cpath)), mirrorpath(""), is_meta_pending(false) + cachepath(SAFESTRPTR(cpath)), mirrorpath(""), is_meta_pending(false), holding_mtime(-1) { pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); @@ -524,9 +524,9 @@ int FdEntity::Open(headers_t* pmeta, off_t size, time_t time, bool no_fd_lock_wa size_orgmeta = 0; } - // set mtime(set "x-amz-meta-mtime" in orgmeta) + // set mtime and ctime(set "x-amz-meta-mtime" and "x-amz-meta-ctime" in orgmeta) if(-1 != time){ - if(0 != SetMtime(time, /*lock_already_held=*/ true)){ + if(0 != SetMCtime(time, time, /*lock_already_held=*/ true)){ S3FS_PRN_ERR("failed to set mtime. errno(%d)", errno); fclose(pfile); pfile = NULL; @@ -657,7 +657,7 @@ int FdEntity::SetCtime(time_t time, bool lock_already_held) return 0; } -int FdEntity::SetMtime(time_t time, bool lock_already_held) +int FdEntity::SetAtime(time_t time, bool lock_already_held) { AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); @@ -666,12 +666,28 @@ int FdEntity::SetMtime(time_t time, bool lock_already_held) if(-1 == time){ return 0; } + orgmeta["x-amz-meta-atime"] = str(time); + return 0; +} + +// [NOTE] +// This method updates mtime as well as ctime. +// +int FdEntity::SetMCtime(time_t mtime, time_t ctime, bool lock_already_held) +{ + AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); + + S3FS_PRN_INFO3("[path=%s][fd=%d][mtime=%lld][ctime=%lld]", path.c_str(), fd, static_cast(mtime), static_cast(ctime)); + + if(mtime < 0 || ctime < 0){ + return 0; + } if(-1 != fd){ struct timeval tv[2]; - tv[0].tv_sec = time; + tv[0].tv_sec = mtime; tv[0].tv_usec= 0L; - tv[1].tv_sec = tv[0].tv_sec; + tv[1].tv_sec = ctime; tv[1].tv_usec= 0L; if(-1 == futimes(fd, tv)){ S3FS_PRN_ERR("futimes failed. errno(%d)", errno); @@ -679,16 +695,16 @@ int FdEntity::SetMtime(time_t time, bool lock_already_held) } }else if(!cachepath.empty()){ // not opened file yet. - struct utimbuf n_mtime; - n_mtime.modtime = time; - n_mtime.actime = time; - if(-1 == utime(cachepath.c_str(), &n_mtime)){ + struct utimbuf n_time; + n_time.modtime = mtime; + n_time.actime = ctime; + if(-1 == utime(cachepath.c_str(), &n_time)){ S3FS_PRN_ERR("utime failed. errno(%d)", errno); return -errno; } } - orgmeta["x-amz-meta-ctime"] = str(time); - orgmeta["x-amz-meta-mtime"] = str(time); + orgmeta["x-amz-meta-mtime"] = str(mtime); + orgmeta["x-amz-meta-ctime"] = str(ctime); return 0; } @@ -704,15 +720,90 @@ bool FdEntity::UpdateCtime() return true; } -bool FdEntity::UpdateMtime() +bool FdEntity::UpdateAtime() { AutoLock auto_lock(&fdent_lock); struct stat st; if(!GetStats(st, /*lock_already_held=*/ true)){ return false; } - orgmeta["x-amz-meta-ctime"] = str(st.st_ctime); - orgmeta["x-amz-meta-mtime"] = str(st.st_mtime); + orgmeta["x-amz-meta-atime"] = str(st.st_atime); + return true; +} + +bool FdEntity::UpdateMtime(bool clear_holding_mtime) +{ + AutoLock auto_lock(&fdent_lock); + + if(0 <= holding_mtime){ + // [NOTE] + // This conditional statement is very special. + // If you copy a file with "cp -p" etc., utimens or chown will be + // called after opening the file, after that call to write, flush. + // If normally utimens are not called(cases like "cp" only), mtime + // should be updated at the file flush. + // Here, check the holding_mtime value to prevent mtime from being + // overwritten. + // + if(clear_holding_mtime){ + if(!ClearHoldingMtime(true)){ + return false; + } + } + }else{ + struct stat st; + if(!GetStats(st, /*lock_already_held=*/ true)){ + return false; + } + orgmeta["x-amz-meta-mtime"] = str(st.st_mtime); + } + return true; +} + +bool FdEntity::SetHoldingMtime(time_t mtime, bool lock_already_held) +{ + AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); + + if(mtime < 0){ + return false; + } + holding_mtime = mtime; + return true; +} + +bool FdEntity::ClearHoldingMtime(bool lock_already_held) +{ + AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); + + if(holding_mtime < 0){ + return false; + } + struct stat st; + if(!GetStats(st, true)){ + return false; + } + if(-1 != fd){ + struct timeval tv[2]; + tv[0].tv_sec = holding_mtime; + tv[0].tv_usec= 0L; + tv[1].tv_sec = st.st_ctime; + tv[1].tv_usec= 0L; + if(-1 == futimes(fd, tv)){ + S3FS_PRN_ERR("futimes failed. errno(%d)", errno); + return false; + } + }else if(!cachepath.empty()){ + // not opened file yet. + struct utimbuf n_time; + n_time.modtime = holding_mtime; + n_time.actime = st.st_ctime; + if(-1 == utime(cachepath.c_str(), &n_time)){ + S3FS_PRN_ERR("utime failed. errno(%d)", errno); + return false; + } + } + holding_mtime = -1; + return true; } @@ -1469,14 +1560,16 @@ bool FdEntity::MergeOrgMeta(headers_t& updatemeta) } updatemeta = orgmeta; orgmeta.erase("x-amz-copy-source"); - // update ctime/mtime - time_t updatetime = get_mtime(updatemeta, false); // not overcheck - if(0 != updatetime){ - SetMtime(updatetime, true); + + // update ctime/mtime/atime + time_t mtime = get_mtime(updatemeta, false); // not overcheck + time_t ctime = get_ctime(updatemeta, false); // not overcheck + time_t atime = get_atime(updatemeta, false); // not overcheck + if(0 <= mtime){ + SetMCtime(mtime, (ctime < 0 ? mtime : ctime), true); } - updatetime = get_ctime(updatemeta, false); // not overcheck - if(0 != updatetime){ - SetCtime(updatetime, true); + if(0 <= atime){ + SetAtime(atime, true); } is_meta_pending |= !upload_id.empty(); diff --git a/src/fdcache_entity.h b/src/fdcache_entity.h index 7a7115d..0e16244 100644 --- a/src/fdcache_entity.h +++ b/src/fdcache_entity.h @@ -51,7 +51,8 @@ class FdEntity std::string cachepath; // local cache file path // (if this is empty, does not load/save pagelist.) std::string mirrorpath; // mirror file path to local cache file path - volatile bool is_meta_pending; + volatile bool is_meta_pending; + volatile time_t holding_mtime; // if mtime is updated while the file is open, it is set time_t value private: static int FillFile(int fd, unsigned char byte, off_t size, off_t start); @@ -85,9 +86,14 @@ class FdEntity bool GetStats(struct stat& st, bool lock_already_held = false); int SetCtime(time_t time, bool lock_already_held = false); - int SetMtime(time_t time, bool lock_already_held = false); + int SetAtime(time_t time, bool lock_already_held = false); + int SetMCtime(time_t mtime, time_t ctime, bool lock_already_held = false); bool UpdateCtime(); - bool UpdateMtime(); + bool UpdateAtime(); + bool UpdateMtime(bool clear_holding_mtime = false); + bool UpdateMCtime(); + bool SetHoldingMtime(time_t mtime, bool lock_already_held = false); + bool ClearHoldingMtime(bool lock_already_held = false); bool GetSize(off_t& size); bool GetXattr(std::string& xattr); bool SetXattr(const std::string& xattr); diff --git a/src/metaheader.cpp b/src/metaheader.cpp index 15fa898..eb1b67d 100644 --- a/src/metaheader.cpp +++ b/src/metaheader.cpp @@ -31,7 +31,7 @@ //------------------------------------------------------------------- // Utility functions for convert //------------------------------------------------------------------- -time_t get_mtime(const char *str) +static time_t cvt_string_to_time(const char *str) { // [NOTE] // In rclone, there are cases where ns is set to x-amz-meta-mtime @@ -54,37 +54,49 @@ static time_t get_time(const headers_t& meta, const char *header) { headers_t::const_iterator iter; if(meta.end() == (iter = meta.find(header))){ - return 0; + return -1; } - return get_mtime((*iter).second.c_str()); + return cvt_string_to_time((*iter).second.c_str()); } time_t get_mtime(const headers_t& meta, bool overcheck) { time_t t = get_time(meta, "x-amz-meta-mtime"); - if(t != 0){ + if(0 < t){ return t; } t = get_time(meta, "x-amz-meta-goog-reserved-file-mtime"); - if(t != 0){ + if(0 < t){ return t; } if(overcheck){ return get_lastmodified(meta); } - return 0; + return -1; } time_t get_ctime(const headers_t& meta, bool overcheck) { time_t t = get_time(meta, "x-amz-meta-ctime"); - if(t != 0){ + if(0 < t){ return t; } if(overcheck){ return get_lastmodified(meta); } - return 0; + return -1; +} + +time_t get_atime(const headers_t& meta, bool overcheck) +{ + time_t t = get_time(meta, "x-amz-meta-atime"); + if(0 < t){ + return t; + } + if(overcheck){ + return get_lastmodified(meta); + } + return -1; } off_t get_size(const char *s) @@ -244,7 +256,7 @@ time_t get_lastmodified(const char* s) { struct tm tm; if(!s){ - return 0L; + return -1; } memset(&tm, 0, sizeof(struct tm)); strptime(s, "%a, %d %b %Y %H:%M:%S %Z", &tm); @@ -255,7 +267,7 @@ time_t get_lastmodified(const headers_t& meta) { headers_t::const_iterator iter = meta.find("Last-Modified"); if(meta.end() == iter){ - return 0; + return -1; } return get_lastmodified((*iter).second.c_str()); } @@ -276,6 +288,8 @@ bool is_need_check_obj_detail(const headers_t& meta) // if the object has x-amz-meta information, checking is no more. if(meta.end() != meta.find("x-amz-meta-mode") || meta.end() != meta.find("x-amz-meta-mtime") || + meta.end() != meta.find("x-amz-meta-ctime") || + meta.end() != meta.find("x-amz-meta-atime") || meta.end() != meta.find("x-amz-meta-uid") || meta.end() != meta.find("x-amz-meta-gid") || meta.end() != meta.find("x-amz-meta-owner") || diff --git a/src/metaheader.h b/src/metaheader.h index 1022eeb..726e640 100644 --- a/src/metaheader.h +++ b/src/metaheader.h @@ -40,9 +40,9 @@ typedef std::map headers_t; //------------------------------------------------------------------- // Functions //------------------------------------------------------------------- -time_t get_mtime(const char *s); time_t get_mtime(const headers_t& meta, bool overcheck = true); time_t get_ctime(const headers_t& meta, bool overcheck = true); +time_t get_atime(const headers_t& meta, bool overcheck = true); off_t get_size(const char *s); off_t get_size(const headers_t& meta); mode_t get_mode(const char *s, int base = 0); diff --git a/src/s3fs.cpp b/src/s3fs.cpp index e290ee4..106d1d5 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -125,10 +125,10 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, static int directory_empty(const char* path); static int rename_large_object(const char* from, const char* to); static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gid); -static int create_directory_object(const char* path, mode_t mode, time_t time, uid_t uid, gid_t gid); -static int rename_object(const char* from, const char* to); -static int rename_object_nocopy(const char* from, const char* to); -static int clone_directory_object(const char* from, const char* to); +static int create_directory_object(const char* path, mode_t mode, time_t atime, time_t mtime, time_t ctime, uid_t uid, gid_t gid); +static int rename_object(const char* from, const char* to, bool update_ctime); +static int rename_object_nocopy(const char* from, const char* to, bool update_ctime); +static int clone_directory_object(const char* from, const char* to, bool update_ctime); static int rename_directory(const char* from, const char* to); static int remote_mountpath_exists(const char* path); static void free_xattrs(xattrs_t& xattrs); @@ -751,7 +751,19 @@ int put_headers(const char* path, headers_t& meta, bool is_copy, bool update_mti } if(ent){ time_t mtime = get_mtime(meta); - ent->SetMtime(mtime); + time_t ctime = get_ctime(meta); + time_t atime = get_atime(meta); + if(mtime < 0){ + mtime = 0L; + } + if(ctime < 0){ + ctime = 0L; + } + if(atime < 0){ + atime = 0L; + } + ent->SetMCtime(mtime, ctime); + ent->SetAtime(atime); } } return 0; @@ -910,6 +922,7 @@ static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gi meta["x-amz-meta-uid"] = str(uid); meta["x-amz-meta-gid"] = str(gid); meta["x-amz-meta-mode"] = str(mode); + meta["x-amz-meta-atime"] = str(now); meta["x-amz-meta-ctime"] = str(now); meta["x-amz-meta-mtime"] = str(now); @@ -985,9 +998,9 @@ static int s3fs_create(const char* _path, mode_t mode, struct fuse_file_info* fi return 0; } -static int create_directory_object(const char* path, mode_t mode, time_t time, uid_t uid, gid_t gid) +static int create_directory_object(const char* path, mode_t mode, time_t atime, time_t mtime, time_t ctime, uid_t uid, gid_t gid) { - S3FS_PRN_INFO1("[path=%s][mode=%04o][time=%lld][uid=%u][gid=%u]", path, mode, static_cast(time), (unsigned int)uid, (unsigned int)gid); + S3FS_PRN_INFO1("[path=%s][mode=%04o][atime=%lld][mtime=%lld][ctime=%lld][uid=%u][gid=%u]", path, mode, static_cast(atime), static_cast(ctime), static_cast(mtime), (unsigned int)uid, (unsigned int)gid); if(!path || '\0' == path[0]){ return -1; @@ -1001,8 +1014,9 @@ static int create_directory_object(const char* path, mode_t mode, time_t time, u meta["x-amz-meta-uid"] = str(uid); meta["x-amz-meta-gid"] = str(gid); meta["x-amz-meta-mode"] = str(mode); - meta["x-amz-meta-ctime"] = str(time); - meta["x-amz-meta-mtime"] = str(time); + meta["x-amz-meta-atime"] = str(atime); + meta["x-amz-meta-mtime"] = str(mtime); + meta["x-amz-meta-ctime"] = str(ctime); S3fsCurl s3fscurl; return s3fscurl.PutRequest(tpath.c_str(), meta, -1); // fd=-1 means for creating zero byte object. @@ -1030,8 +1044,8 @@ static int s3fs_mkdir(const char* _path, mode_t mode) } return result; } + result = create_directory_object(path, mode, time(NULL), time(NULL), time(NULL), pcxt->uid, pcxt->gid); - result = create_directory_object(path, mode, time(NULL), pcxt->uid, pcxt->gid); StatCache::getStatCacheData()->DelStat(path); S3FS_MALLOCTRIM(0); @@ -1156,6 +1170,7 @@ static int s3fs_symlink(const char* _from, const char* _to) headers_t headers; headers["Content-Type"] = std::string("application/octet-stream"); // Static headers["x-amz-meta-mode"] = str(S_IFLNK | S_IRWXU | S_IRWXG | S_IRWXO); + headers["x-amz-meta-atime"] = str(now); headers["x-amz-meta-ctime"] = str(now); headers["x-amz-meta-mtime"] = str(now); headers["x-amz-meta-uid"] = str(pcxt->uid); @@ -1192,7 +1207,7 @@ static int s3fs_symlink(const char* _from, const char* _to) return result; } -static int rename_object(const char* from, const char* to) +static int rename_object(const char* from, const char* to, bool update_ctime) { int result; std::string s3_realpath; @@ -1213,6 +1228,9 @@ static int rename_object(const char* from, const char* to) } s3_realpath = get_realpath(from); + if(update_ctime){ + meta["x-amz-meta-ctime"] = str(time(NULL)); + } meta["x-amz-copy-source"] = urlEncode(service_path + bucket + s3_realpath); meta["Content-Type"] = S3fsCurl::LookupMimeType(std::string(to)); meta["x-amz-metadata-directive"] = "REPLACE"; @@ -1232,7 +1250,7 @@ static int rename_object(const char* from, const char* to) return result; } -static int rename_object_nocopy(const char* from, const char* to) +static int rename_object_nocopy(const char* from, const char* to, bool update_ctime) { int result; @@ -1262,6 +1280,11 @@ static int rename_object_nocopy(const char* from, const char* to) return -EIO; } + // update ctime + if(update_ctime){ + ent->SetCtime(time(NULL)); + } + // upload if(0 != (result = ent->RowFlush(to, true))){ S3FS_PRN_ERR("could not upload file(%s): result=%d", to, result); @@ -1315,7 +1338,7 @@ static int rename_large_object(const char* from, const char* to) return result; } -static int clone_directory_object(const char* from, const char* to) +static int clone_directory_object(const char* from, const char* to, bool update_ctime) { int result = -1; struct stat stbuf; @@ -1326,7 +1349,8 @@ static int clone_directory_object(const char* from, const char* to) if(0 != (result = get_object_attribute(from, &stbuf))){ return result; } - result = create_directory_object(to, stbuf.st_mode, stbuf.st_mtime, stbuf.st_uid, stbuf.st_gid); + result = create_directory_object(to, stbuf.st_mode, stbuf.st_atime, stbuf.st_mtime, (update_ctime ? time(NULL) : stbuf.st_ctime), stbuf.st_uid, stbuf.st_gid); + StatCache::getStatCacheData()->DelStat(to); return result; @@ -1424,7 +1448,11 @@ static int rename_directory(const char* from, const char* to) // rename directory objects. for(mn_cur = mn_head; mn_cur; mn_cur = mn_cur->next){ if(mn_cur->is_dir && mn_cur->old_path && '\0' != mn_cur->old_path[0]){ - if(0 != (result = clone_directory_object(mn_cur->old_path, mn_cur->new_path))){ + // [NOTE] + // The ctime is updated only for the top (from) directory. + // Other than that, it will not be updated. + // + if(0 != (result = clone_directory_object(mn_cur->old_path, mn_cur->new_path, (strfrom == mn_cur->old_path)))){ S3FS_PRN_ERR("clone_directory_object returned an error(%d)", result); free_mvnodes(mn_head); return -EIO; @@ -1436,11 +1464,10 @@ static int rename_directory(const char* from, const char* to) // does a safe copy - copies first and then deletes old for(mn_cur = mn_head; mn_cur; mn_cur = mn_cur->next){ if(!mn_cur->is_dir){ - // TODO: call s3fs_rename instead? if(!nocopyapi && !norenameapi){ - result = rename_object(mn_cur->old_path, mn_cur->new_path); + result = rename_object(mn_cur->old_path, mn_cur->new_path, false); // keep ctime }else{ - result = rename_object_nocopy(mn_cur->old_path, mn_cur->new_path); + result = rename_object_nocopy(mn_cur->old_path, mn_cur->new_path, false); // keep ctime } if(0 != result){ S3FS_PRN_ERR("rename_object returned an error(%d)", result); @@ -1511,9 +1538,9 @@ static int s3fs_rename(const char* _from, const char* _to) result = rename_large_object(from, to); }else{ if(!nocopyapi && !norenameapi){ - result = rename_object(from, to); + result = rename_object(from, to, true); // update ctime }else{ - result = rename_object_nocopy(from, to); + result = rename_object_nocopy(from, to, true); // update ctime } } S3FS_MALLOCTRIM(0); @@ -1575,7 +1602,7 @@ static int s3fs_chmod(const char* _path, mode_t mode) StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") - if(0 != (result = create_directory_object(newpath.c_str(), mode, stbuf.st_mtime, stbuf.st_uid, stbuf.st_gid))){ + if(0 != (result = create_directory_object(newpath.c_str(), mode, stbuf.st_atime, stbuf.st_mtime, time(NULL), stbuf.st_uid, stbuf.st_gid))){ return result; } }else{ @@ -1668,7 +1695,7 @@ static int s3fs_chmod_nocopy(const char* _path, mode_t mode) StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") - if(0 != (result = create_directory_object(newpath.c_str(), mode, stbuf.st_mtime, stbuf.st_uid, stbuf.st_gid))){ + if(0 != (result = create_directory_object(newpath.c_str(), mode, stbuf.st_atime, stbuf.st_mtime, time(NULL), stbuf.st_uid, stbuf.st_gid))){ return result; } }else{ @@ -1751,7 +1778,7 @@ static int s3fs_chown(const char* _path, uid_t uid, gid_t gid) StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") - if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_mtime, uid, gid))){ + if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_atime, stbuf.st_mtime, time(NULL), uid, gid))){ return result; } }else{ @@ -1851,7 +1878,7 @@ static int s3fs_chown_nocopy(const char* _path, uid_t uid, gid_t gid) StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") - if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_mtime, uid, gid))){ + if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_atime, stbuf.st_mtime, time(NULL), uid, gid))){ return result; } }else{ @@ -1894,7 +1921,7 @@ static int s3fs_utimens(const char* _path, const struct timespec ts[2]) struct stat stbuf; dirtype nDirType = DIRTYPE_UNKNOWN; - S3FS_PRN_INFO("[path=%s][mtime=%lld]", path, static_cast(ts[1].tv_sec)); + S3FS_PRN_INFO("[path=%s][mtime=%lld][ctime/atime=%lld]", path, static_cast(ts[1].tv_sec), static_cast(ts[0].tv_sec)); if(0 == strcmp(path, "/")){ S3FS_PRN_ERR("Could not change mtime for mount point."); @@ -1931,12 +1958,14 @@ static int s3fs_utimens(const char* _path, const struct timespec ts[2]) StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") - if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, ts[1].tv_sec, stbuf.st_uid, stbuf.st_gid))){ + if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, ts[0].tv_sec, ts[1].tv_sec, ts[0].tv_sec, stbuf.st_uid, stbuf.st_gid))){ return result; } }else{ headers_t updatemeta; updatemeta["x-amz-meta-mtime"] = str(ts[1].tv_sec); + updatemeta["x-amz-meta-ctime"] = str(ts[0].tv_sec); + updatemeta["x-amz-meta-atime"] = str(ts[0].tv_sec); updatemeta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; @@ -1961,6 +1990,13 @@ static int s3fs_utimens(const char* _path, const struct timespec ts[2]) return -EIO; } StatCache::getStatCacheData()->DelStat(nowcache); + + // [NOTE] + // Depending on the order in which write/flush and utimens are called, + // the mtime updated here may be overwritten at the time of flush. + // To avoid that, set a special flag. + // + ent->SetHoldingMtime(ts[1].tv_sec); // ts[1].tv_sec is mtime } }else{ // not opened file, then put headers @@ -1986,7 +2022,7 @@ static int s3fs_utimens_nocopy(const char* _path, const struct timespec ts[2]) struct stat stbuf; dirtype nDirType = DIRTYPE_UNKNOWN; - S3FS_PRN_INFO1("[path=%s][mtime=%lld]", path, static_cast(ts[1].tv_sec)); + S3FS_PRN_INFO1("[path=%s][mtime=%lld][atime/ctime=%lld]", path, static_cast(ts[1].tv_sec), static_cast(ts[0].tv_sec)); if(0 == strcmp(path, "/")){ S3FS_PRN_ERR("Could not change mtime for mount point."); @@ -2024,7 +2060,7 @@ static int s3fs_utimens_nocopy(const char* _path, const struct timespec ts[2]) StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") - if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, ts[1].tv_sec, stbuf.st_uid, stbuf.st_gid))){ + if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, ts[0].tv_sec, ts[1].tv_sec, ts[0].tv_sec, stbuf.st_uid, stbuf.st_gid))){ return result; } }else{ @@ -2038,9 +2074,15 @@ static int s3fs_utimens_nocopy(const char* _path, const struct timespec ts[2]) return -EIO; } - // set mtime - if(0 != (result = ent->SetMtime(ts[1].tv_sec))){ - S3FS_PRN_ERR("could not set mtime to file(%s): result=%d", strpath.c_str(), result); + // set mtime/ctime + if(0 != (result = ent->SetMCtime(ts[1].tv_sec, ts[0].tv_sec))){ + S3FS_PRN_ERR("could not set mtime and ctime to file(%s): result=%d", strpath.c_str(), result); + return result; + } + + // set atime + if(0 != (result = ent->SetAtime(ts[0].tv_sec))){ + S3FS_PRN_ERR("could not set atime to file(%s): result=%d", strpath.c_str(), result); return result; } @@ -2279,7 +2321,8 @@ static int s3fs_flush(const char* _path, struct fuse_file_info* fi) AutoFdEntity autoent; FdEntity* ent; if(NULL != (ent = autoent.ExistOpen(path, static_cast(fi->fh)))){ - ent->UpdateMtime(); + ent->UpdateMtime(true); // clear the flag not to update mtime. + ent->UpdateCtime(); result = ent->Flush(false); } S3FS_MALLOCTRIM(0); @@ -2302,6 +2345,7 @@ static int s3fs_fsync(const char* _path, int datasync, struct fuse_file_info* fi if(NULL != (ent = autoent.ExistOpen(path, static_cast(fi->fh)))){ if(0 == datasync){ ent->UpdateMtime(); + ent->UpdateCtime(); } result = ent->Flush(false); } @@ -2878,7 +2922,7 @@ static int s3fs_setxattr(const char* path, const char* name, const char* value, StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") - if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_mtime, stbuf.st_uid, stbuf.st_gid))){ + if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_atime, stbuf.st_mtime, stbuf.st_ctime, stbuf.st_uid, stbuf.st_gid))){ return result; } @@ -3162,7 +3206,7 @@ static int s3fs_removexattr(const char* path, const char* name) StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") - if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_mtime, stbuf.st_uid, stbuf.st_gid))){ + if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, stbuf.st_atime, stbuf.st_mtime, stbuf.st_ctime, stbuf.st_uid, stbuf.st_gid))){ free_xattrs(xattrs); return result; } diff --git a/test/integration-test-main.sh b/test/integration-test-main.sh index 052ec7f..4af2d58 100755 --- a/test/integration-test-main.sh +++ b/test/integration-test-main.sh @@ -586,52 +586,261 @@ function test_mtime_file { function test_update_time() { describe "Testing update time function ..." + # # create the test + # mk_test_file + base_atime=`get_atime $TEST_TEXT_FILE` + base_ctime=`get_ctime $TEST_TEXT_FILE` + base_mtime=`get_mtime $TEST_TEXT_FILE` + sleep 2 + + # + # chmod -> update only ctime + # + chmod +x $TEST_TEXT_FILE + atime=`get_atime $TEST_TEXT_FILE` ctime=`get_ctime $TEST_TEXT_FILE` mtime=`get_mtime $TEST_TEXT_FILE` - - sleep 2 - chmod +x $TEST_TEXT_FILE - - ctime2=`get_ctime $TEST_TEXT_FILE` - mtime2=`get_mtime $TEST_TEXT_FILE` - if [ $ctime -eq $ctime2 -o $mtime -ne $mtime2 ]; then - echo "Expected updated ctime: $ctime != $ctime2 and same mtime: $mtime == $mtime2" + if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then + echo "chmod expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi - + base_ctime=$ctime sleep 2 + + # + # chown -> update only ctime + # chown $UID $TEST_TEXT_FILE - - ctime3=`get_ctime $TEST_TEXT_FILE` - mtime3=`get_mtime $TEST_TEXT_FILE` - if [ $ctime2 -eq $ctime3 -o $mtime2 -ne $mtime3 ]; then - echo "Expected updated ctime: $ctime2 != $ctime3 and same mtime: $mtime2 == $mtime3" + atime=`get_atime $TEST_TEXT_FILE` + ctime=`get_ctime $TEST_TEXT_FILE` + mtime=`get_mtime $TEST_TEXT_FILE` + if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then + echo "chown expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi - + base_ctime=$ctime sleep 2 + + # + # set_xattr -> update only ctime + # set_xattr key value $TEST_TEXT_FILE - - ctime4=`get_ctime $TEST_TEXT_FILE` - mtime4=`get_mtime $TEST_TEXT_FILE` - if [ $ctime3 -eq $ctime4 -o $mtime3 -ne $mtime4 ]; then - echo "Expected updated ctime: $ctime3 != $ctime4 and same mtime: $mtime3 == $mtime4" + atime=`get_atime $TEST_TEXT_FILE` + ctime=`get_ctime $TEST_TEXT_FILE` + mtime=`get_mtime $TEST_TEXT_FILE` + if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then + echo "set_xattr expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi - + base_ctime=$ctime sleep 2 - echo foo >> $TEST_TEXT_FILE - ctime5=`get_ctime $TEST_TEXT_FILE` - mtime5=`get_mtime $TEST_TEXT_FILE` - if [ $ctime4 -eq $ctime5 -o $mtime4 -eq $mtime5 ]; then - echo "Expected updated ctime: $ctime4 != $ctime5 and updated mtime: $mtime4 != $mtime5" + # + # touch -> update ctime/atime/mtime + # + touch $TEST_TEXT_FILE + atime=`get_atime $TEST_TEXT_FILE` + ctime=`get_ctime $TEST_TEXT_FILE` + mtime=`get_mtime $TEST_TEXT_FILE` + if [ $base_atime -eq $atime -o $base_ctime -eq $ctime -o $base_mtime -eq $mtime ]; then + echo "touch expected updated ctime: $base_ctime != $ctime, mtime: $base_mtime != $mtime, atime: $base_atime != $atime" + return 1 + fi + base_atime=$atime + base_mtime=$mtime + base_ctime=$ctime + sleep 2 + + # + # "touch -a" -> update ctime/atime, not update mtime + # + touch -a $TEST_TEXT_FILE + atime=`get_atime $TEST_TEXT_FILE` + ctime=`get_ctime $TEST_TEXT_FILE` + mtime=`get_mtime $TEST_TEXT_FILE` + if [ $base_atime -eq $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then + echo "touch with -a option expected updated ctime: $base_ctime != $ctime, atime: $base_atime != $atime and same mtime: $base_mtime == $mtime" + return 1 + fi + base_atime=$atime + base_ctime=$ctime + sleep 2 + + # + # append -> update ctime/mtime, not update atime + # + echo foo >> $TEST_TEXT_FILE + atime=`get_atime $TEST_TEXT_FILE` + ctime=`get_ctime $TEST_TEXT_FILE` + mtime=`get_mtime $TEST_TEXT_FILE` + if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -eq $mtime ]; then + echo "append expected updated ctime: $base_ctime != $ctime, mtime: $base_mtime != $mtime and same atime: $base_atime == $atime" + return 1 + fi + base_mtime=$mtime + base_ctime=$ctime + sleep 2 + + # + # cp -p -> update ctime, not update atime/mtime + # + TIME_TEST_TEXT_FILE=test-s3fs-time.txt + cp -p $TEST_TEXT_FILE $TIME_TEST_TEXT_FILE + atime=`get_atime $TIME_TEST_TEXT_FILE` + ctime=`get_ctime $TIME_TEST_TEXT_FILE` + mtime=`get_mtime $TIME_TEST_TEXT_FILE` + if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then + echo "cp with -p option expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" + return 1 + fi + sleep 2 + + # + # mv -> update ctime, not update atime/mtime + # + TIME2_TEST_TEXT_FILE=test-s3fs-time2.txt + mv $TEST_TEXT_FILE $TIME2_TEST_TEXT_FILE + atime=`get_atime $TIME2_TEST_TEXT_FILE` + ctime=`get_ctime $TIME2_TEST_TEXT_FILE` + mtime=`get_mtime $TIME2_TEST_TEXT_FILE` + if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then + echo "mv expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi - rm_test_file + rm_test_file $TIME_TEST_TEXT_FILE + rm_test_file $TIME2_TEST_TEXT_FILE +} + +function test_update_directory_time() { + describe "Testing update time for directory function ..." + + # + # create the directory and sub-directory and a file in directory + # + TIME_TEST_SUBDIR="$TEST_DIR/testsubdir" + TIME_TEST_FILE_INDIR="$TEST_DIR/testfile" + mk_test_dir + mkdir $TIME_TEST_SUBDIR + touch $TIME_TEST_FILE_INDIR + + base_atime=`get_atime $TEST_DIR` + base_ctime=`get_ctime $TEST_DIR` + base_mtime=`get_mtime $TEST_DIR` + subdir_atime=`get_atime $TIME_TEST_SUBDIR` + subdir_ctime=`get_ctime $TIME_TEST_SUBDIR` + subdir_mtime=`get_mtime $TIME_TEST_SUBDIR` + subfile_atime=`get_atime $TIME_TEST_FILE_INDIR` + subfile_ctime=`get_ctime $TIME_TEST_FILE_INDIR` + subfile_mtime=`get_mtime $TIME_TEST_FILE_INDIR` + sleep 2 + + # + # chmod -> update only ctime + # + chmod 0777 $TEST_DIR + atime=`get_atime $TEST_DIR` + ctime=`get_ctime $TEST_DIR` + mtime=`get_mtime $TEST_DIR` + if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then + echo "chmod expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" + return 1 + fi + base_ctime=$ctime + sleep 2 + + # + # chown -> update only ctime + # + chown $UID $TEST_DIR + atime=`get_atime $TEST_DIR` + ctime=`get_ctime $TEST_DIR` + mtime=`get_mtime $TEST_DIR` + if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then + echo "chown expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" + return 1 + fi + base_ctime=$ctime + sleep 2 + + # + # set_xattr -> update only ctime + # + set_xattr key value $TEST_DIR + atime=`get_atime $TEST_DIR` + ctime=`get_ctime $TEST_DIR` + mtime=`get_mtime $TEST_DIR` + if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then + echo "set_xattr expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" + return 1 + fi + base_ctime=$ctime + sleep 2 + + # + # touch -> update ctime/atime/mtime + # + touch $TEST_DIR + atime=`get_atime $TEST_DIR` + ctime=`get_ctime $TEST_DIR` + mtime=`get_mtime $TEST_DIR` + if [ $base_atime -eq $atime -o $base_ctime -eq $ctime -o $base_mtime -eq $mtime ]; then + echo "touch expected updated ctime: $base_ctime != $ctime, mtime: $base_mtime != $mtime, atime: $base_atime != $atime" + return 1 + fi + base_atime=$atime + base_mtime=$mtime + base_ctime=$ctime + sleep 2 + + # + # "touch -a" -> update ctime/atime, not update mtime + # + touch -a $TEST_DIR + atime=`get_atime $TEST_DIR` + ctime=`get_ctime $TEST_DIR` + mtime=`get_mtime $TEST_DIR` + if [ $base_atime -eq $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then + echo "touch with -a option expected updated ctime: $base_ctime != $ctime, atime: $base_atime != $atime and same mtime: $base_mtime == $mtime" + return 1 + fi + base_atime=$atime + base_ctime=$ctime + sleep 2 + + # + # mv -> update ctime, not update atime/mtime for taget directory + # not update any for sub-directory and a file + # + TIME_TEST_DIR=timetestdir + TIME2_TEST_SUBDIR="$TIME_TEST_DIR/testsubdir" + TIME2_TEST_FILE_INDIR="$TIME_TEST_DIR/testfile" + mv $TEST_DIR $TIME_TEST_DIR + atime=`get_atime $TIME_TEST_DIR` + ctime=`get_ctime $TIME_TEST_DIR` + mtime=`get_mtime $TIME_TEST_DIR` + if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then + echo "mv expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" + return 1 + fi + atime=`get_atime $TIME2_TEST_SUBDIR` + ctime=`get_ctime $TIME2_TEST_SUBDIR` + mtime=`get_mtime $TIME2_TEST_SUBDIR` + if [ $subdir_atime -ne $atime -o $subdir_ctime -ne $ctime -o $subdir_mtime -ne $mtime ]; then + echo "mv for sub-directory expected same ctime: $subdir_ctime == $ctime, mtime: $subdir_mtime == $mtime, atime: $subdir_atime == $atime" + return 1 + fi + atime=`get_atime $TIME2_TEST_FILE_INDIR` + ctime=`get_ctime $TIME2_TEST_FILE_INDIR` + mtime=`get_mtime $TIME2_TEST_FILE_INDIR` + if [ $subfile_atime -ne $atime -o $subfile_ctime -ne $ctime -o $subfile_mtime -ne $mtime ]; then + echo "mv for a file in directory expected same ctime: $subfile_ctime == $ctime, mtime: $subfile_mtime == $mtime, atime: $subfile_atime == $atime" + return 1 + fi + + rm -r $TIME_TEST_DIR } function test_rm_rf_dir { @@ -1080,6 +1289,7 @@ function add_all_tests { add_tests test_extended_attributes add_tests test_mtime_file add_tests test_update_time + add_tests test_update_directory_time add_tests test_rm_rf_dir add_tests test_copy_file add_tests test_write_after_seek_ahead diff --git a/test/test-utils.sh b/test/test-utils.sh index 7103e8b..c77209a 100644 --- a/test/test-utils.sh +++ b/test/test-utils.sh @@ -269,6 +269,14 @@ function get_mtime() { fi } +function get_atime() { + if [ `uname` = "Darwin" ]; then + stat -f "%a" "$1" + else + stat -c "%X" "$1" + fi +} + function get_permissions() { if [ `uname` = "Darwin" ]; then stat -f "%p" "$1"