youngkingdom/s3fs-fuse
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed bugs in removing xattrs func and and test for it

Browse Source
This commit is contained in:
Takeshi Nakatani
2025-06-10 16:26:11 +00:00
committed by Andrew Gaul
parent b624596685
commit 97659c41f2
4 changed files with 106 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/cache.cpp
View File

@ -411,14 +411,21 @@ bool StatCache::UpdateMetaStats(const std::string& key, const headers_t& meta)
// update only meta keys
for(auto metaiter = meta.cbegin(); metaiter != meta.cend(); ++metaiter){
auto tag = CaseInsensitiveStringView(metaiter->first);
const auto& value = metaiter->second;
if(tag == "content-type" ||
tag == "content-length" ||
tag == "etag" ||
tag == "last-modified" ||
tag.is_prefix("x-amz")){
ent->meta[metaiter->first] = value;
if(metaiter->second.empty()){
auto metaiter2 = ent->meta.find(metaiter->first);
if(metaiter2 != ent->meta.cend()){
ent->meta.erase(metaiter2);
}
}else{
auto tag = CaseInsensitiveStringView(metaiter->first);
if(tag == "content-type" ||
tag == "content-length" ||
tag == "etag" ||
tag == "last-modified" ||
tag.is_prefix("x-amz") )
{
ent->meta[metaiter->first] = metaiter->second;
}
}
}

27
src/s3fs.cpp
View File

@ -4048,7 +4048,13 @@ static int s3fs_removexattr(const char* path, const char* name)
if(nullptr != (ent = autoent.OpenExistFdEntity(path))){
if(ent->MergeOrgMeta(updatemeta)){
// meta is changed, but now uploading.
// then the meta is pending and accumulated to be put after the upload is complete.
//
// [NOTE]
// Then the meta is pending and accumulated to be put after the upload is complete.
// If there is no need to hold it, need_put_header will remain true.
// Please note that if "x-amz-meta-xattr" in updatemeta is empty, it will be deleted
// by ent->MergeOrgMeta.
//
S3FS_PRN_INFO("meta pending until upload is complete");
need_put_header = false;
@ -4057,11 +4063,28 @@ static int s3fs_removexattr(const char* path, const char* name)
}
}
if(need_put_header){
// not found opened file.
// not found opened file or not have pending meta
//
// [NOTE]
// If "x-amz-meta-xattr" exists in updatemeta but its value is empty,
// this key will be deleted here.
// After that, if the "x-amz-meta-xattr" key does not exist,
// "x-amz-meta-xattr" will also be deleted from meta.
//
// Please note that ent->MergeOrgMeta() may be called before this
// process. In ent->MergeOrgMeta(), if "x-amz-meta-xattr" exists in
// updatemeta but its value is empty, it will be deleted first.
//
auto iter = updatemeta.find("x-amz-meta-xattr");
if(iter != updatemeta.end() && iter->second.empty()){
updatemeta.erase(iter);
}
if(updatemeta.end() == updatemeta.find("x-amz-meta-xattr")){
auto miter = meta.find("x-amz-meta-xattr");
if(miter != meta.end()){
meta.erase(miter);
}
}
merge_headers(meta, updatemeta, true);

66
test/integration-test-main.sh
View File

@ -467,8 +467,8 @@ function test_update_metadata_external_small_object() {
local TEST_CHMOD_FILE="${TEST_TEXT_FILE}_chmod.${TEST_FILE_EXT}"
local TEST_CHOWN_FILE="${TEST_TEXT_FILE}_chown.${TEST_FILE_EXT}"
local TEST_UTIMENS_FILE="${TEST_TEXT_FILE}_utimens.${TEST_FILE_EXT}"
local TEST_SETXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}"
local TEST_RMXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}"
local TEST_SETXATTR_FILE="${TEST_TEXT_FILE}_set_xattr.${TEST_FILE_EXT}"
local TEST_RMXATTR_FILE="${TEST_TEXT_FILE}_rm_xattr.${TEST_FILE_EXT}"
local TEST_INPUT="TEST_STRING_IN_SMALL_FILE"
@ -503,16 +503,37 @@ function test_update_metadata_external_small_object() {
echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}"
set_xattr key value "${TEST_SETXATTR_FILE}"
cmp "${TEST_SETXATTR_FILE}" <(echo "${TEST_INPUT}")
XATTR_VALUE=$(get_xattr key "${TEST_SETXATTR_FILE}")
if [ -z "${XATTR_VALUE}" ] || [ "${XATTR_VALUE}" != "value" ]; then
echo "could not read xattr(key) value."
return 1
fi
#
# remove xattr
#
# "%7B%22key%22%3A%22dmFsdWU%3D%22%7D" = {"key":"value"}
#
OBJECT_NAME=$(basename "${PWD}")/"${TEST_RMXATTR_FILE}"
echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --metadata xattr=%7B%22key%22%3A%22dmFsdWU%3D%22%7D
del_xattr key "${TEST_RMXATTR_FILE}"
cmp "${TEST_RMXATTR_FILE}" <(echo "${TEST_INPUT}")
# [FIXME]
# For macos, the xattrs value specified with the "--metadata" option cannot
# be set on the object. We confirmed this with the following versions:
# aws-cli/2.27.35 Python/3.13.3 Darwin/22.6.0 exe/x86_64
# We also tried to run "aws s3api put-object", but the result was the same.
#
# Since xattrs cannot be set on objects uploaded with the aws command, this
# will be skipped for macos.
# If a solution is found in the future, we will test it on macos as well.
#
if ! uname | grep -q Darwin; then
OBJECT_NAME=$(basename "${PWD}")/"${TEST_RMXATTR_FILE}"
echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --metadata xattr=%7B%22key%22%3A%22dmFsdWU%3D%22%7D
del_xattr key "${TEST_RMXATTR_FILE}"
cmp "${TEST_RMXATTR_FILE}" <(echo "${TEST_INPUT}")
if find_xattr key "${TEST_RMXATTR_FILE}"; then
echo "could read xattr(key) value after removing it."
return 1
fi
fi
rm -f "${TEST_CHMOD_FILE}"
rm -f "${TEST_CHOWN_FILE}"
@ -531,8 +552,8 @@ function test_update_metadata_external_large_object() {
local TEST_CHMOD_FILE="${TEST_TEXT_FILE}_chmod.${TEST_FILE_EXT}"
local TEST_CHOWN_FILE="${TEST_TEXT_FILE}_chown.${TEST_FILE_EXT}"
local TEST_UTIMENS_FILE="${TEST_TEXT_FILE}_utimens.${TEST_FILE_EXT}"
local TEST_SETXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}"
local TEST_RMXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}"
local TEST_SETXATTR_FILE="${TEST_TEXT_FILE}_set_xattr.${TEST_FILE_EXT}"
local TEST_RMXATTR_FILE="${TEST_TEXT_FILE}_rm_xattr.${TEST_FILE_EXT}"
../../junk_data $((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) > "${TEMP_DIR}/${BIG_FILE}"
@ -567,16 +588,37 @@ function test_update_metadata_external_large_object() {
aws_cli s3 cp "${TEMP_DIR}/${BIG_FILE}" "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress
set_xattr key value "${TEST_SETXATTR_FILE}"
cmp "${TEST_SETXATTR_FILE}" "${TEMP_DIR}/${BIG_FILE}"
XATTR_VALUE=$(get_xattr key "${TEST_SETXATTR_FILE}")
if [ -z "${XATTR_VALUE}" ] || [ "${XATTR_VALUE}" != "value" ]; then
echo "could not read xattr(key) value."
return 1
fi
#
# remove xattr
#
# "%7B%22key%22%3A%22dmFsdWU%3D%22%7D" = {"key":"value"}
#
OBJECT_NAME=$(basename "${PWD}")/"${TEST_RMXATTR_FILE}"
aws_cli s3 cp "${TEMP_DIR}/${BIG_FILE}" "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress --metadata xattr=%7B%22key%22%3A%22dmFsdWU%3D%22%7D
del_xattr key "${TEST_RMXATTR_FILE}"
cmp "${TEST_RMXATTR_FILE}" "${TEMP_DIR}/${BIG_FILE}"
# [FIXME]
# For macos, the xattrs value specified with the "--metadata" option cannot
# be set on the object. We confirmed this with the following versions:
# aws-cli/2.27.35 Python/3.13.3 Darwin/22.6.0 exe/x86_64
# We also tried to run "aws s3api put-object", but the result was the same.
#
# Since xattrs cannot be set on objects uploaded with the aws command, this
# will be skipped for macos.
# If a solution is found in the future, we will test it on macos as well.
#
if ! uname | grep -q Darwin; then
OBJECT_NAME=$(basename "${PWD}")/"${TEST_RMXATTR_FILE}"
aws_cli s3 cp "${TEMP_DIR}/${BIG_FILE}" "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress --metadata xattr=%7B%22key%22%3A%22dmFsdWU%3D%22%7D
del_xattr key "${TEST_RMXATTR_FILE}"
cmp "${TEST_RMXATTR_FILE}" "${TEMP_DIR}/${BIG_FILE}"
if find_xattr key "${TEST_RMXATTR_FILE}"; then
echo "could read xattr(key) value after removing it."
return 1
fi
fi
rm -f "${TEMP_DIR}/${BIG_FILE}"
rm -f "${TEST_CHMOD_FILE}"

12
test/test-utils.sh
View File

@ -78,6 +78,18 @@ else
STAT_BIN=(stat)
fi
function find_xattr() {
if [ "$(uname)" = "Darwin" ]; then
local LIST_XATTRS_KEYVALS; LIST_XATTRS_KEYVALS=$(xattr -l "$2" 2>/dev/null)
if ! echo "${LIST_XATTRS_KEYVALS}" | grep -q "$1"; then
return 1
fi
else
getfattr --absolute-names -n "$1" "$2" >/dev/null 2>&1 || return 1
fi
return 0
}
function get_xattr() {
if [ "$(uname)" = "Darwin" ]; then
xattr -p "$1" "$2"