diff --git a/doc/man/s3fs.1 b/doc/man/s3fs.1 index 27dc029..73e9088 100644 --- a/doc/man/s3fs.1 +++ b/doc/man/s3fs.1 @@ -305,6 +305,10 @@ If this option is specified with nocopyapi, then s3fs ignores it. \fB\-o\fR use_path_request_style (use legacy API calling style) Enable compatibility with S3-like APIs which do not support the virtual-host request style, by using the older path request style. .TP +\fB\-o\fR listobjectsv2 (use ListObjectsV2) +Issue ListObjectsV2 instead of ListObjects, useful on object +stores without ListObjects support. +.TP \fB\-o\fR noua (suppress User-Agent header) Usually s3fs outputs of the User-Agent in "s3fs/ (commit hash ; )" format. If this option is specified, s3fs suppresses the output of the User-Agent. diff --git a/src/curl.cpp b/src/curl.cpp index 0ddc3a3..76ca0dc 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -136,6 +136,7 @@ off_t S3fsCurl::multipart_size = MULTIPART_SIZE; // default off_t S3fsCurl::multipart_copy_size = 512 * 1024 * 1024; // default signature_type_t S3fsCurl::signature_type = V2_OR_V4; // default bool S3fsCurl::is_ua = true; // default +bool S3fsCurl::listobjectsv2 = false; // default bool S3fsCurl::is_use_session_token= false; // default bool S3fsCurl::requester_pays = false; // default @@ -2666,7 +2667,7 @@ void S3fsCurl::insertV2Headers() std::string turl; std::string server_path = type == REQTYPE_LISTBUCKET ? "/" : path; MakeUrlResource(server_path.c_str(), resource, turl); - if(!query_string.empty() && type != REQTYPE_LISTBUCKET){ + if(!query_string.empty() && type != REQTYPE_CHKBUCKET && type != REQTYPE_LISTBUCKET){ resource += "?" + query_string; } @@ -3365,10 +3366,16 @@ int S3fsCurl::CheckBucket() if(!CreateCurlHandle()){ return -EIO; } + std::string urlargs; + if(S3fsCurl::IsListObjectsV2()){ + query_string = "list-type=2"; + urlargs = "?" + query_string; + } std::string resource; std::string turl; MakeUrlResource(get_realpath("/").c_str(), resource, turl); + turl += urlargs; url = prepare_url(turl.c_str()); path = get_realpath("/"); requestHeaders = NULL; diff --git a/src/curl.h b/src/curl.h index dbffbac..ad760fb 100644 --- a/src/curl.h +++ b/src/curl.h @@ -164,6 +164,7 @@ class S3fsCurl static off_t multipart_copy_size; static signature_type_t signature_type; static bool is_ua; // User-Agent + static bool listobjectsv2; static bool requester_pays; // variables @@ -359,6 +360,8 @@ class S3fsCurl static bool SetUserAgentFlag(bool isset) { bool bresult = S3fsCurl::is_ua; S3fsCurl::is_ua = isset; return bresult; } static bool IsUserAgentFlag() { return S3fsCurl::is_ua; } static void InitUserAgent(); + static bool SetListObjectsV2(bool isset) { bool bresult = S3fsCurl::listobjectsv2; S3fsCurl::listobjectsv2 = isset; return bresult; } + static bool IsListObjectsV2() { return S3fsCurl::listobjectsv2; } static bool SetRequesterPays(bool flag) { bool old_flag = S3fsCurl::requester_pays; S3fsCurl::requester_pays = flag; return old_flag; } static bool IsRequesterPays() { return S3fsCurl::requester_pays; } static bool SetIMDSVersion(int version); diff --git a/src/s3fs.cpp b/src/s3fs.cpp index cb6b22e..9f66200 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -2642,6 +2642,7 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, std::string query_delimiter; std::string query_prefix; std::string query_maxkey; + std::string next_continuation_token; std::string next_marker; bool truncated = true; S3fsCurl s3fscurl; @@ -2672,7 +2673,16 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, } while(truncated){ - std::string each_query = query_delimiter; + // append parameters to query in alphabetical order + std::string each_query = ""; + if(!next_continuation_token.empty()){ + each_query += "continuation-token=" + urlEncode(next_continuation_token) + "&"; + next_continuation_token = ""; + } + each_query += query_delimiter; + if(S3fsCurl::IsListObjectsV2()){ + each_query += "list-type=2&"; + } if(!next_marker.empty()){ each_query += "marker=" + urlEncode(next_marker) + "&"; next_marker = ""; @@ -2699,11 +2709,16 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, return -EIO; } if(true == (truncated = is_truncated(doc))){ - xmlChar* tmpch = get_next_marker(doc); - if(tmpch){ + xmlChar* tmpch; + if(NULL != (tmpch = get_next_contination_token(doc))){ + next_continuation_token = (char*)tmpch; + xmlFree(tmpch); + }else if(NULL != (tmpch = get_next_marker(doc))){ next_marker = (char*)tmpch; xmlFree(tmpch); - }else{ + } + + if(next_continuation_token.empty() && next_marker.empty()){ // If did not specify "delimiter", s3 did not return "NextMarker". // On this case, can use last name for next marker. // @@ -4634,6 +4649,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar S3fsCurl::SetUserAgentFlag(false); return 0; } + if(0 == strcmp(arg, "listobjectsv2")){ + S3fsCurl::SetListObjectsV2(true); + return 0; + } if(0 == strcmp(arg, "use_xattr")){ is_use_xattr = true; return 0; diff --git a/src/s3fs_help.cpp b/src/s3fs_help.cpp index df0b2ea..d8fdbaf 100644 --- a/src/s3fs_help.cpp +++ b/src/s3fs_help.cpp @@ -382,6 +382,10 @@ static const char help_string[] = " the virtual-host request style, by using the older path request\n" " style.\n" "\n" + " listobjectsv2 (use ListObjectsV2)\n" + " Issue ListObjectsV2 instead of ListObjects, useful on object\n" + " stores without ListObjects support.\n" + "\n" " noua (suppress User-Agent header)\n" " Usually s3fs outputs of the User-Agent in \"s3fs/ (commit\n" " hash ; )\" format.\n" diff --git a/src/s3fs_xml.cpp b/src/s3fs_xml.cpp index 8847eea..f65d084 100644 --- a/src/s3fs_xml.cpp +++ b/src/s3fs_xml.cpp @@ -109,6 +109,11 @@ static xmlChar* get_prefix(xmlDocPtr doc) return get_base_exp(doc, "Prefix"); } +xmlChar* get_next_contination_token(xmlDocPtr doc) +{ + return get_base_exp(doc, "NextContinuationToken"); +} + xmlChar* get_next_marker(xmlDocPtr doc) { return get_base_exp(doc, "NextMarker"); diff --git a/src/s3fs_xml.h b/src/s3fs_xml.h index d029720..7b2f8c6 100644 --- a/src/s3fs_xml.h +++ b/src/s3fs_xml.h @@ -36,6 +36,7 @@ bool is_truncated(xmlDocPtr doc); int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx, const char* ex_contents, const char* ex_key, const char* ex_etag, int isCPrefix, S3ObjList& head); int append_objects_from_xml(const char* path, xmlDocPtr doc, S3ObjList& head); +xmlChar* get_next_contination_token(xmlDocPtr doc); xmlChar* get_next_marker(xmlDocPtr doc); bool get_incomp_mpu_list(xmlDocPtr doc, incomp_mpu_list_t& list);