diff --git a/src/fdcache_entity.cpp b/src/fdcache_entity.cpp index 15043c5..fa4dc6c 100644 --- a/src/fdcache_entity.cpp +++ b/src/fdcache_entity.cpp @@ -1581,7 +1581,7 @@ bool FdEntity::MergeOrgMeta(headers_t& updatemeta) } // global function in s3fs.cpp -int put_headers(const char* path, headers_t& meta, bool is_copy, bool update_mtime); +int put_headers(const char* path, headers_t& meta, bool is_copy); int FdEntity::UploadPendingMeta() { @@ -1593,7 +1593,7 @@ int FdEntity::UploadPendingMeta() headers_t updatemeta = orgmeta; updatemeta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(path.c_str())); // put headers, no need to update mtime to avoid dead lock - int result = put_headers(path.c_str(), updatemeta, true, false); + int result = put_headers(path.c_str(), updatemeta, true); if(0 != result){ S3FS_PRN_ERR("failed to put header after flushing file(%s) by(%d).", path.c_str(), result); } diff --git a/src/s3fs.cpp b/src/s3fs.cpp index 9cf23d3..cb6b22e 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -107,7 +107,7 @@ static const std::string aws_secretkey = "AWSSecretKey"; //------------------------------------------------------------------- // Global functions : prototype //------------------------------------------------------------------- -int put_headers(const char* path, headers_t& meta, bool is_copy, bool update_mtime = true); // [NOTE] global function because this is called from FdEntity class +int put_headers(const char* path, headers_t& meta, bool is_copy); // [NOTE] global function because this is called from FdEntity class //------------------------------------------------------------------- // Static functions : prototype @@ -737,7 +737,7 @@ static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char* // ow_sse_flg is for over writing sse header by use_sse option. // @return fuse return code // -int put_headers(const char* path, headers_t& meta, bool is_copy, bool update_mtime) +int put_headers(const char* path, headers_t& meta, bool is_copy) { int result; S3fsCurl s3fscurl(true); @@ -759,37 +759,6 @@ int put_headers(const char* path, headers_t& meta, bool is_copy, bool update_mti return result; } } - - // [NOTE] - // if path is 'dir/', it does not have cache(could not open file for directory stat) - // - if(update_mtime && '/' != path[strlen(path) - 1] ){ - AutoFdEntity autoent; - FdEntity* ent; - if(NULL == (ent = autoent.ExistOpen(path, -1, !FdManager::IsCacheDir()))){ - // no opened fd - if(FdManager::IsCacheDir()){ - // create cache file if be needed - ent = autoent.Open(path, &meta, buf.st_size, -1, false, true); - } - } - if(ent){ - time_t mtime = get_mtime(meta); - time_t ctime = get_ctime(meta); - time_t atime = get_atime(meta); - if(mtime < 0){ - mtime = 0L; - } - if(ctime < 0){ - ctime = 0L; - } - if(atime < 0){ - atime = 0L; - } - ent->SetMCtime(mtime, ctime); - ent->SetAtime(atime); - } - } return 0; } @@ -1294,7 +1263,7 @@ static int rename_object(const char* from, const char* to, bool update_ctime) } // copy - if(0 != (result = put_headers(to, meta, true, false))){ + if(0 != (result = put_headers(to, meta, true))){ return result; } @@ -1694,7 +1663,7 @@ static int s3fs_chmod(const char* _path, mode_t mode) merge_headers(meta, updatemeta, true); // upload meta directly. - if(0 != (result = put_headers(strpath.c_str(), meta, true, false))){ + if(0 != (result = put_headers(strpath.c_str(), meta, true))){ return result; } StatCache::getStatCacheData()->DelStat(nowcache); @@ -1867,7 +1836,7 @@ static int s3fs_chown(const char* _path, uid_t uid, gid_t gid) merge_headers(meta, updatemeta, true); // upload meta directly. - if(0 != (result = put_headers(strpath.c_str(), meta, true, false))){ + if(0 != (result = put_headers(strpath.c_str(), meta, true))){ return result; } StatCache::getStatCacheData()->DelStat(nowcache); @@ -2055,7 +2024,7 @@ static int s3fs_utimens(const char* _path, const struct timespec ts[2]) merge_headers(meta, updatemeta, true); // upload meta directly. - if(0 != (result = put_headers(strpath.c_str(), meta, true, false))){ + if(0 != (result = put_headers(strpath.c_str(), meta, true))){ return result; } StatCache::getStatCacheData()->DelStat(nowcache); @@ -3017,9 +2986,8 @@ static int s3fs_setxattr(const char* path, const char* name, const char* value, // AutoFdEntity autoent; FdEntity* ent; + bool need_put_header = true; if(NULL != (ent = autoent.ExistOpen(path, -1, true))){ - // the file is opened now. - // get xattr and make new xattr std::string strxattr; if(ent->GetXattr(strxattr)){ @@ -3035,31 +3003,26 @@ static int s3fs_setxattr(const char* path, const char* name, const char* value, } if(ent->MergeOrgMeta(updatemeta)){ - // now uploading - // the meta is pending and accumulated to be put after the upload is complete. + // meta is changed, but now uploading. + // then the meta is pending and accumulated to be put after the upload is complete. S3FS_PRN_INFO("meta pending until upload is complete"); - }else{ - // allow to put header - // updatemeta already merged the orgmeta of the opened files. - if(0 != (result = put_headers(strpath.c_str(), updatemeta, true))){ - return result; - } - StatCache::getStatCacheData()->DelStat(nowcache); + need_put_header = false; } - }else{ - // not opened file, then put headers - merge_headers(meta, updatemeta, true); - - // NOTICE: modify xattr from base meta + } + if(need_put_header){ + // not found opened file. if(0 != (result = set_xattrs_to_header(meta, name, value, size, flags))){ return result; } + merge_headers(meta, updatemeta, true); + // upload meta directly. if(0 != (result = put_headers(strpath.c_str(), meta, true))){ return result; } StatCache::getStatCacheData()->DelStat(nowcache); } + return 0; } @@ -3307,29 +3270,24 @@ static int s3fs_removexattr(const char* path, const char* name) // AutoFdEntity autoent; FdEntity* ent; + bool need_put_header = true; if(NULL != (ent = autoent.ExistOpen(path, -1, true))){ - // the file is opened now. if(ent->MergeOrgMeta(updatemeta)){ - // now uploading - // the meta is pending and accumulated to be put after the upload is complete. + // meta is changed, but now uploading. + // then the meta is pending and accumulated to be put after the upload is complete. S3FS_PRN_INFO("meta pending until upload is complete"); - }else{ - // allow to put header - // updatemeta already merged the orgmeta of the opened files. - if(updatemeta["x-amz-meta-xattr"].empty()){ - updatemeta.erase("x-amz-meta-xattr"); - } - if(0 != (result = put_headers(strpath.c_str(), updatemeta, true))){ - return result; - } - StatCache::getStatCacheData()->DelStat(nowcache); + need_put_header = false; } - }else{ - // not opened file, then put headers + } + if(need_put_header){ + // not found opened file. if(updatemeta["x-amz-meta-xattr"].empty()){ updatemeta.erase("x-amz-meta-xattr"); } + merge_headers(meta, updatemeta, true); + + // upload meta directly. if(0 != (result = put_headers(strpath.c_str(), meta, true))){ return result; } diff --git a/test/integration-test-main.sh b/test/integration-test-main.sh index add8872..843b890 100755 --- a/test/integration-test-main.sh +++ b/test/integration-test-main.sh @@ -354,6 +354,8 @@ function test_update_metadata_external_small_object() { TEST_CHMOD_FILE="${TEST_TEXT_FILE}_chmod.${TEST_FILE_EXT}" TEST_CHOWN_FILE="${TEST_TEXT_FILE}_chown.${TEST_FILE_EXT}" TEST_UTIMENS_FILE="${TEST_TEXT_FILE}_utimens.${TEST_FILE_EXT}" + TEST_SETXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}" + TEST_RMXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}" TEST_INPUT="TEST_STRING_IN_SMALL_FILE" @@ -381,9 +383,29 @@ function test_update_metadata_external_small_object() { touch ${TEST_UTIMENS_FILE} cmp ${TEST_UTIMENS_FILE} <(echo "${TEST_INPUT}") + # + # set xattr + # + OBJECT_NAME="$(basename $PWD)/${TEST_SETXATTR_FILE}" + echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" + set_xattr key value ${TEST_SETXATTR_FILE} + cmp ${TEST_SETXATTR_FILE} <(echo "${TEST_INPUT}") + + # + # remove xattr + # + # "%7B%22key%22%3A%22dmFsdWU%3D%22%7D" = {"key":"value"} + # + OBJECT_NAME="$(basename $PWD)/${TEST_RMXATTR_FILE}" + echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --metadata xattr=%7B%22key%22%3A%22dmFsdWU%3D%22%7D + del_xattr key ${TEST_RMXATTR_FILE} + cmp ${TEST_RMXATTR_FILE} <(echo "${TEST_INPUT}") + rm -f ${TEST_CHMOD_FILE} rm -f ${TEST_CHOWN_FILE} rm -f ${TEST_UTIMENS_FILE} + rm -f ${TEST_SETXATTR_FILE} + rm -f ${TEST_RMXATTR_FILE} } function test_update_metadata_external_large_object() { @@ -396,6 +418,8 @@ function test_update_metadata_external_large_object() { TEST_CHMOD_FILE="${TEST_TEXT_FILE}_chmod.${TEST_FILE_EXT}" TEST_CHOWN_FILE="${TEST_TEXT_FILE}_chown.${TEST_FILE_EXT}" TEST_UTIMENS_FILE="${TEST_TEXT_FILE}_utimens.${TEST_FILE_EXT}" + TEST_SETXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}" + TEST_RMXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}" dd if=/dev/urandom of="${TEMP_DIR}/${BIG_FILE}" bs=$BIG_FILE_BLOCK_SIZE count=$BIG_FILE_COUNT @@ -423,10 +447,30 @@ function test_update_metadata_external_large_object() { touch ${TEST_UTIMENS_FILE} cmp ${TEST_UTIMENS_FILE} ${TEMP_DIR}/${BIG_FILE} + # + # set xattr + # + OBJECT_NAME="$(basename $PWD)/${TEST_SETXATTR_FILE}" + aws_cli s3 cp ${TEMP_DIR}/${BIG_FILE} "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress + set_xattr key value ${TEST_SETXATTR_FILE} + cmp ${TEST_SETXATTR_FILE} ${TEMP_DIR}/${BIG_FILE} + + # + # remove xattr + # + # "%7B%22key%22%3A%22dmFsdWU%3D%22%7D" = {"key":"value"} + # + OBJECT_NAME="$(basename $PWD)/${TEST_RMXATTR_FILE}" + aws_cli s3 cp ${TEMP_DIR}/${BIG_FILE} "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress --metadata xattr=%7B%22key%22%3A%22dmFsdWU%3D%22%7D + del_xattr key ${TEST_RMXATTR_FILE} + cmp ${TEST_RMXATTR_FILE} ${TEMP_DIR}/${BIG_FILE} + rm -f ${TEMP_DIR}/${BIG_FILE} rm -f ${TEST_CHMOD_FILE} rm -f ${TEST_CHOWN_FILE} rm -f ${TEST_UTIMENS_FILE} + rm -f ${TEST_SETXATTR_FILE} + rm -f ${TEST_RMXATTR_FILE} } function test_rename_before_close {