diff --git a/src/curl.cpp b/src/curl.cpp index cb308d7..7188824 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -1219,6 +1219,7 @@ S3fsCurl* S3fsCurl::UploadMultipartPostRetryCallback(S3fsCurl* s3fscurl) if(!get_keyword_value(s3fscurl->url, "uploadId", upload_id)){ return NULL; } + upload_id = urlDecode(upload_id); // decode if(!get_keyword_value(s3fscurl->url, "partNumber", part_num_str)){ return NULL; } @@ -1266,6 +1267,7 @@ S3fsCurl* S3fsCurl::CopyMultipartPostRetryCallback(S3fsCurl* s3fscurl) if(!get_keyword_value(s3fscurl->url, "uploadId", upload_id)){ return NULL; } + upload_id = urlDecode(upload_id); // decode if(!get_keyword_value(s3fscurl->url, "partNumber", part_num_str)){ return NULL; } @@ -2789,7 +2791,7 @@ std::string S3fsCurl::CalcSignature(const std::string& method, const std::string requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", access_token.c_str()); } - uriencode = urlEncode(canonical_uri); + uriencode = urlEncodePath(canonical_uri); StringCQ = method + "\n"; if(method == "HEAD" || method == "PUT" || method == "DELETE"){ StringCQ += uriencode + "\n"; @@ -2798,11 +2800,11 @@ std::string S3fsCurl::CalcSignature(const std::string& method, const std::string }else if(method == "GET" && is_prefix(uriencode.c_str(), "/")){ StringCQ += uriencode +"\n"; }else if(method == "GET" && !is_prefix(uriencode.c_str(), "/")){ - StringCQ += "/\n" + urlEncode2(canonical_uri) +"\n"; + StringCQ += "/\n" + urlEncodeQuery(canonical_uri) +"\n"; }else if(method == "POST"){ StringCQ += uriencode + "\n"; } - StringCQ += urlEncode2(query_string) + "\n"; + StringCQ += urlEncodeQuery(query_string) + "\n"; StringCQ += get_canonical_headers(requestHeaders) + "\n"; StringCQ += get_sorted_header_keys(requestHeaders) + "\n"; StringCQ += payload_hash; @@ -3921,7 +3923,11 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, const std::string& std::string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); - query_string = "uploadId=" + upload_id; + // [NOTE] + // Encode the upload_id here. + // In compatible S3 servers(Cloudflare, etc), there are cases where characters that require URL encoding are included. + // + query_string = "uploadId=" + urlEncodeGeneral(upload_id); turl += "?" + query_string; url = prepare_url(turl.c_str()); path = get_realpath(tpath); @@ -4039,7 +4045,11 @@ int S3fsCurl::AbortMultipartUpload(const char* tpath, const std::string& upload_ std::string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); - query_string = "uploadId=" + upload_id; + // [NOTE] + // Encode the upload_id here. + // In compatible S3 servers(Cloudflare, etc), there are cases where characters that require URL encoding are included. + // + query_string = "uploadId=" + urlEncodeGeneral(upload_id); turl += "?" + query_string; url = prepare_url(turl.c_str()); path = get_realpath(tpath); @@ -4101,7 +4111,12 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, const st } // make request - query_string = "partNumber=" + str(part_num) + "&uploadId=" + upload_id; + // + // [NOTE] + // Encode the upload_id here. + // In compatible S3 servers(Cloudflare, etc), there are cases where characters that require URL encoding are included. + // + query_string = "partNumber=" + str(part_num) + "&uploadId=" + urlEncodeGeneral(upload_id); std::string urlargs = "?" + query_string; std::string resource; std::string turl; @@ -4169,7 +4184,11 @@ int S3fsCurl::CopyMultipartPostSetup(const char* from, const char* to, int part_ if(!from || !to){ return -EINVAL; } - query_string = "partNumber=" + str(part_num) + "&uploadId=" + upload_id; + // [NOTE] + // Encode the upload_id here. + // In compatible S3 servers(Cloudflare, etc), there are cases where characters that require URL encoding are included. + // + query_string = "partNumber=" + str(part_num) + "&uploadId=" + urlEncodeGeneral(upload_id); std::string urlargs = "?" + query_string; std::string resource; std::string turl; diff --git a/src/curl_util.cpp b/src/curl_util.cpp index 50cfeb1..08bbf84 100644 --- a/src/curl_util.cpp +++ b/src/curl_util.cpp @@ -247,7 +247,7 @@ bool MakeUrlResource(const char* realpath, std::string& resourcepath, std::strin if(!realpath){ return false; } - resourcepath = urlEncode(service_path + S3fsCred::GetBucket() + realpath); + resourcepath = urlEncodePath(service_path + S3fsCred::GetBucket() + realpath); url = s3host + resourcepath; return true; } diff --git a/src/fdcache_entity.cpp b/src/fdcache_entity.cpp index c205988..0d8dbdd 100644 --- a/src/fdcache_entity.cpp +++ b/src/fdcache_entity.cpp @@ -2410,7 +2410,7 @@ int FdEntity::UploadPending(int fd, AutoLock::Type type) }else if(UPDATE_META_PENDING == pending_status){ headers_t updatemeta = orgmeta; - updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(path.c_str())); + updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(path.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; // put headers, no need to update mtime to avoid dead lock diff --git a/src/s3fs.cpp b/src/s3fs.cpp index 9a54a3a..72b8fd5 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -1364,7 +1364,7 @@ static int rename_object(const char* from, const char* to, bool update_ctime) if(update_ctime){ meta["x-amz-meta-ctime"] = s3fs_str_realtime(); } - meta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); + meta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); meta["Content-Type"] = S3fsCurl::LookupMimeType(std::string(to)); meta["x-amz-metadata-directive"] = "REPLACE"; @@ -1845,7 +1845,7 @@ static int s3fs_chmod(const char* _path, mode_t mode) headers_t updatemeta; updatemeta["x-amz-meta-ctime"] = s3fs_str_realtime(); updatemeta["x-amz-meta-mode"] = str(mode); - updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); + updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; // check opened file handle. @@ -2050,7 +2050,7 @@ static int s3fs_chown(const char* _path, uid_t uid, gid_t gid) updatemeta["x-amz-meta-ctime"] = s3fs_str_realtime(); updatemeta["x-amz-meta-uid"] = str(uid); updatemeta["x-amz-meta-gid"] = str(gid); - updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); + updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; // check opened file handle. @@ -2285,7 +2285,7 @@ static int update_mctime_parent_directory(const char* _path) updatemeta["x-amz-meta-mtime"] = str(mctime); updatemeta["x-amz-meta-ctime"] = str(mctime); updatemeta["x-amz-meta-atime"] = str(atime); - updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); + updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; merge_headers(meta, updatemeta, true); @@ -2378,7 +2378,7 @@ static int s3fs_utimens(const char* _path, const struct timespec ts[2]) updatemeta["x-amz-meta-mtime"] = str(mtime); updatemeta["x-amz-meta-ctime"] = str(ctime); updatemeta["x-amz-meta-atime"] = str(atime); - updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); + updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; // check opened file handle. @@ -3268,9 +3268,9 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, s3_realpath = get_realpath(path); if(s3_realpath.empty() || '/' != *s3_realpath.rbegin()){ // last word must be "/" - query_prefix += urlEncode(s3_realpath.substr(1) + "/"); + query_prefix += urlEncodePath(s3_realpath.substr(1) + "/"); }else{ - query_prefix += urlEncode(s3_realpath.substr(1)); + query_prefix += urlEncodePath(s3_realpath.substr(1)); } if (check_content_only){ // Just need to know if there are child objects in dir @@ -3284,7 +3284,7 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, // append parameters to query in alphabetical order std::string each_query; if(!next_continuation_token.empty()){ - each_query += "continuation-token=" + urlEncode(next_continuation_token) + "&"; + each_query += "continuation-token=" + urlEncodePath(next_continuation_token) + "&"; next_continuation_token = ""; } each_query += query_delimiter; @@ -3292,7 +3292,7 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, each_query += "list-type=2&"; } if(!next_marker.empty()){ - each_query += "marker=" + urlEncode(next_marker) + "&"; + each_query += "marker=" + urlEncodePath(next_marker) + "&"; next_marker = ""; } each_query += query_maxkey; @@ -3488,7 +3488,7 @@ static bool build_inherited_xattr_value(const char* path, std::string& xattrvalu raw_xattr_value += parent_default_value; raw_xattr_value += "\"}"; - xattrvalue = urlEncode(raw_xattr_value); + xattrvalue = urlEncodePath(raw_xattr_value); return true; } @@ -3596,7 +3596,7 @@ static std::string build_xattrs(const xattrs_t& xattrs) if(strxattrs.empty()){ strxattrs = "{}"; } - strxattrs = urlEncode(strxattrs); + strxattrs = urlEncodePath(strxattrs); return strxattrs; } @@ -3733,7 +3733,7 @@ static int s3fs_setxattr(const char* path, const char* name, const char* value, std::string strSourcePath = (mount_prefix.empty() && "/" == strpath) ? "//" : strpath; headers_t updatemeta; updatemeta["x-amz-meta-ctime"] = s3fs_str_realtime(); - updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); + updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; // check opened file handle. @@ -4028,7 +4028,7 @@ static int s3fs_removexattr(const char* path, const char* name) // set xattr all object std::string strSourcePath = (mount_prefix.empty() && "/" == strpath) ? "//" : strpath; headers_t updatemeta; - updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); + updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; if(!xattrs.empty()){ updatemeta["x-amz-meta-xattr"] = build_xattrs(xattrs); diff --git a/src/string_util.cpp b/src/string_util.cpp index 98423a2..7ee0af1 100644 --- a/src/string_util.cpp +++ b/src/string_util.cpp @@ -147,23 +147,32 @@ std::string trim(const std::string &s, const char *t /* = SPACES */) } // -// urlEncode a fuse path, -// taking into special consideration "/", -// otherwise regular urlEncode. +// Three url encode functions // -std::string urlEncode(const std::string &s) +// urlEncodeGeneral: A general URL encoding function. +// urlEncodePath : A function that URL encodes by excluding the path +// separator('/'). +// urlEncodeQuery : A function that does URL encoding by excluding +// some characters('=', '&' and '%'). +// This function can be used when the target string +// contains already URL encoded strings. It also +// excludes the character () used in query strings. +// Therefore, it is a function to use as URL encoding +// for use in query strings. +// +static const char* encode_general_except_chars = ".-_~"; // For general URL encode +static const char* encode_path_except_chars = ".-_~/"; // For fuse(included path) URL encode +static const char* encode_query_except_chars = ".-_~=&%"; // For query params(and encoded string) + +static std::string rawUrlEncode(const std::string &s, const char* except_chars) { std::string result; for (size_t i = 0; i < s.length(); ++i) { unsigned char c = s[i]; - if (c == '/' // Note- special case for fuse paths... - || c == '.' - || c == '-' - || c == '_' - || c == '~' - || (c >= 'a' && c <= 'z') - || (c >= 'A' && c <= 'Z') - || (c >= '0' && c <= '9')) + if((except_chars && NULL != strchr(except_chars, c)) || + (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') ) { result += c; }else{ @@ -174,34 +183,19 @@ std::string urlEncode(const std::string &s) return result; } -// -// urlEncode a fuse path, -// taking into special consideration "/", -// otherwise regular urlEncode. -// -std::string urlEncode2(const std::string &s) +std::string urlEncodeGeneral(const std::string &s) { - std::string result; - for (size_t i = 0; i < s.length(); ++i) { - unsigned char c = s[i]; - if (c == '=' // Note- special case for fuse paths... - || c == '&' // Note- special case for s3... - || c == '%' - || c == '.' - || c == '-' - || c == '_' - || c == '~' - || (c >= 'a' && c <= 'z') - || (c >= 'A' && c <= 'Z') - || (c >= '0' && c <= '9')) - { - result += c; - }else{ - result += "%"; - result += s3fs_hex_upper(&c, 1); - } - } - return result; + return rawUrlEncode(s, encode_general_except_chars); +} + +std::string urlEncodePath(const std::string &s) +{ + return rawUrlEncode(s, encode_path_except_chars); +} + +std::string urlEncodeQuery(const std::string &s) +{ + return rawUrlEncode(s, encode_query_except_chars); } std::string urlDecode(const std::string& s) diff --git a/src/string_util.h b/src/string_util.h index da25a90..bf6e13b 100644 --- a/src/string_util.h +++ b/src/string_util.h @@ -94,8 +94,9 @@ bool convert_unixtime_from_option_arg(const char* argv, time_t& unixtime); // // For encoding // -std::string urlEncode(const std::string &s); -std::string urlEncode2(const std::string &s); +std::string urlEncodeGeneral(const std::string &s); +std::string urlEncodePath(const std::string &s); +std::string urlEncodeQuery(const std::string &s); std::string urlDecode(const std::string& s); bool takeout_str_dquart(std::string& str);