youngkingdom/s3fs-fuse
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use region instead of endpoint for configuration (#2669)

Browse Source 
This is more consistent with the AWS docs.  Generally endpoint refers
to an HTTP URL not just the region.  Fixes #2668.
This commit is contained in:
Andrew Gaul
2025-05-18 10:49:35 +09:00
committed by GitHub
parent 43f49b15e8
commit 3421025074
7 changed files with 36 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
doc/man/s3fs.1.in
View File

@ -255,12 +255,13 @@ Set a service path when the non-Amazon host requires a prefix.
sets the url to use to access Amazon S3. If you want to use HTTP, then you can set "url=http://s3.amazonaws.com".
If you do not use https, please specify the URL with the url option.
.TP
\fB\-o\fR endpoint (default="us-east-1")
sets the endpoint to use on signature version 4.
\fB\-o\fR region (default="us-east-1")
sets the region to use on signature version 4.
If this option is not specified, s3fs uses "us-east-1" region as the default.
If the s3fs could not connect to the region specified by this option, s3fs could not run.
But if you do not specify this option, and if you can not connect with the default region, s3fs will retry to automatically connect to the other region.
So s3fs can know the correct region name, because s3fs can find it in an error from the S3 server.
You can also specify the legacy -o endpoint which means the same thing.
.TP
\fB\-o\fR sigv2 (default is signature version 4 falling back to version 2)
sets signing AWS requests by using only signature version 2.

2
src/common.h
View File

@ -42,7 +42,7 @@ extern std::string program_name;
extern std::string service_path;
extern std::string s3host;
extern std::string mount_prefix;
extern std::string endpoint;
extern std::string region;
extern std::string cipher_suites;
extern std::string instance_name;

8
src/curl.cpp
View File

@ -2007,7 +2007,7 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
case 301:
case 307:
S3FS_PRN_ERR("HTTP response code 301(Moved Permanently: also happens when bucket's region is incorrect), returning EIO. Body Text: %s", bodydata.c_str());
S3FS_PRN_ERR("The options of url and endpoint may be useful for solving, please try to use both options.");
S3FS_PRN_ERR("The options of url and region may be useful for solving, please try to use both options.");
result = -EIO;
break;
@ -2271,7 +2271,7 @@ std::string S3fsCurl::CalcSignature(const std::string& method, const std::string
unsigned int kDate_len,kRegion_len, kService_len, kSigning_len = 0;
std::unique_ptr<unsigned char[]> kDate = s3fs_HMAC256(kSecret.c_str(), kSecret.size(), reinterpret_cast<const unsigned char*>(strdate.data()), strdate.size(), &kDate_len);
std::unique_ptr<unsigned char[]> kRegion = s3fs_HMAC256(kDate.get(), kDate_len, reinterpret_cast<const unsigned char*>(endpoint.c_str()), endpoint.size(), &kRegion_len);
std::unique_ptr<unsigned char[]> kRegion = s3fs_HMAC256(kDate.get(), kDate_len, reinterpret_cast<const unsigned char*>(region.c_str()), region.size(), &kRegion_len);
std::unique_ptr<unsigned char[]> kService = s3fs_HMAC256(kRegion.get(), kRegion_len, reinterpret_cast<const unsigned char*>("s3"), sizeof("s3") - 1, &kService_len);
std::unique_ptr<unsigned char[]> kSigning = s3fs_HMAC256(kService.get(), kService_len, reinterpret_cast<const unsigned char*>("aws4_request"), sizeof("aws4_request") - 1, &kSigning_len);
@ -2282,7 +2282,7 @@ std::string S3fsCurl::CalcSignature(const std::string& method, const std::string
StringToSign = "AWS4-HMAC-SHA256\n";
StringToSign += date8601 + "\n";
StringToSign += strdate + "/" + endpoint + "/s3/aws4_request\n";
StringToSign += strdate + "/" + region + "/s3/aws4_request\n";
StringToSign += s3fs_hex_lower(sRequest.data(), sRequest.size());
const auto* cscope = reinterpret_cast<const unsigned char*>(StringToSign.c_str());
@ -2382,7 +2382,7 @@ bool S3fsCurl::insertV4Headers(const std::string& access_key_id, const std::stri
if(!S3fsCurl::IsPublicBucket()){
std::string Signature = CalcSignature(op, realpath, query_string + (type == REQTYPE::PREMULTIPOST || type == REQTYPE::MULTILIST ? "=" : ""), strdate, contentSHA256, date8601, secret_access_key, access_token);
std::string auth = "AWS4-HMAC-SHA256 Credential=" + access_key_id + "/" + strdate + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature;
std::string auth = "AWS4-HMAC-SHA256 Credential=" + access_key_id + "/" + strdate + "/" + region + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", auth.c_str());
}

46
src/s3fs.cpp
View File

@ -99,7 +99,7 @@ static bool is_remove_cache = false;
static bool is_use_xattr = false;
static off_t multipart_threshold = 25 * 1024 * 1024;
static int64_t singlepart_copy_limit = 512 * 1024 * 1024;
static bool is_specified_endpoint = false;
static bool is_region_specified = false;
static int s3fs_init_deferred_exit_status = 0;
static bool support_compat_dir = false;// default does not support compatibility directory type
static int max_keys_list_object = 1000;// default is 1000
@ -4243,7 +4243,7 @@ static int s3fs_check_service()
long responseCode = S3fsCurl::S3FSCURL_RESPONSECODE_NOTSET;
std::string responseBody;
if(0 > check_service_request(get_realpath("/"), forceNoSSE, support_compat_dir, responseCode, responseBody)){
// check wrong endpoint, and automatically switch endpoint
// check wrong region, and automatically switch region
if(300 <= responseCode && responseCode < 500){
// check region error(for putting message or retrying)
std::string expectregion;
@ -4252,50 +4252,50 @@ static int s3fs_check_service()
// Check if any case can be retried
if(check_region_error(responseBody.c_str(), responseBody.size(), expectregion)){
// [NOTE]
// If endpoint is not specified(using us-east-1 region) and
// If region is not specified(using us-east-1 region) and
// an error is encountered accessing a different region, we
// will retry the check on the expected region.
// see) https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro
//
if(s3host != "http://s3.amazonaws.com" && s3host != "https://s3.amazonaws.com"){
// specified endpoint for specified url is wrong.
if(is_specified_endpoint){
S3FS_PRN_CRIT("The bucket region is not '%s'(specified) for specified url(%s), it is correctly '%s'. You should specify url(http(s)://s3-%s.amazonaws.com) and endpoint(%s) option.", endpoint.c_str(), s3host.c_str(), expectregion.c_str(), expectregion.c_str(), expectregion.c_str());
// specified region for specified url is wrong.
if(is_region_specified){
S3FS_PRN_CRIT("The bucket region is not '%s'(specified) for specified url(%s), it is correctly '%s'. You should specify url(http(s)://s3-%s.amazonaws.com) and region(%s) option.", region.c_str(), s3host.c_str(), expectregion.c_str(), expectregion.c_str(), expectregion.c_str());
}else{
S3FS_PRN_CRIT("The bucket region is not '%s'(default) for specified url(%s), it is correctly '%s'. You should specify url(http(s)://s3-%s.amazonaws.com) and endpoint(%s) option.", endpoint.c_str(), s3host.c_str(), expectregion.c_str(), expectregion.c_str(), expectregion.c_str());
S3FS_PRN_CRIT("The bucket region is not '%s'(default) for specified url(%s), it is correctly '%s'. You should specify url(http(s)://s3-%s.amazonaws.com) and region(%s) option.", region.c_str(), s3host.c_str(), expectregion.c_str(), expectregion.c_str(), expectregion.c_str());
}
isLoop = false;
}else if(is_specified_endpoint){
// specified endpoint is wrong.
S3FS_PRN_CRIT("The bucket region is not '%s'(specified), it is correctly '%s'. You should specify endpoint(%s) option.", endpoint.c_str(), expectregion.c_str(), expectregion.c_str());
}else if(is_region_specified){
// specified region is wrong.
S3FS_PRN_CRIT("The bucket region is not '%s'(specified), it is correctly '%s'. You should specify region(%s) option.", region.c_str(), expectregion.c_str(), expectregion.c_str());
isLoop = false;
}else if(S3fsCurl::GetSignatureType() == signature_type_t::V4_ONLY || S3fsCurl::GetSignatureType() == signature_type_t::V2_OR_V4){
// current endpoint and url are default value, so try to connect to expected region.
S3FS_PRN_CRIT("Failed to connect region '%s'(default), so retry to connect region '%s' for url(http(s)://s3-%s.amazonaws.com).", endpoint.c_str(), expectregion.c_str(), expectregion.c_str());
// current region and url are default value, so try to connect to expected region.
S3FS_PRN_CRIT("Failed to connect region '%s'(default), so retry to connect region '%s' for url(http(s)://s3-%s.amazonaws.com).", region.c_str(), expectregion.c_str(), expectregion.c_str());
// change endpoint
endpoint = expectregion;
// change region
region = expectregion;
// change url
if(s3host == "http://s3.amazonaws.com"){
s3host = "http://s3-" + endpoint + ".amazonaws.com";
s3host = "http://s3-" + region + ".amazonaws.com";
}else if(s3host == "https://s3.amazonaws.com"){
s3host = "https://s3-" + endpoint + ".amazonaws.com";
s3host = "https://s3-" + region + ".amazonaws.com";
}
}else{
S3FS_PRN_CRIT("The bucket region is not '%s'(default), it is correctly '%s'. You should specify endpoint(%s) option.", endpoint.c_str(), expectregion.c_str(), expectregion.c_str());
S3FS_PRN_CRIT("The bucket region is not '%s'(default), it is correctly '%s'. You should specify region(%s) option.", region.c_str(), expectregion.c_str(), expectregion.c_str());
isLoop = false;
}
}else if(check_endpoint_error(responseBody.c_str(), responseBody.size(), expectendpoint)){
// redirect error
if(pathrequeststyle){
S3FS_PRN_CRIT("S3 service returned PermanentRedirect (current is url(%s) and endpoint(%s)). You need to specify correct url(http(s)://s3-<endpoint>.amazonaws.com) and endpoint option with use_path_request_style option.", s3host.c_str(), endpoint.c_str());
S3FS_PRN_CRIT("S3 service returned PermanentRedirect (current is url(%s) and region(%s)). You need to specify correct url(http(s)://s3-<region>.amazonaws.com) and region option with use_path_request_style option.", s3host.c_str(), region.c_str());
}else{
S3FS_PRN_CRIT("S3 service returned PermanentRedirect with %s (current is url(%s) and endpoint(%s)). You need to specify correct endpoint option.", expectendpoint.c_str(), s3host.c_str(), endpoint.c_str());
S3FS_PRN_CRIT("S3 service returned PermanentRedirect with %s (current is url(%s) and region(%s)). You need to specify correct region option.", expectendpoint.c_str(), s3host.c_str(), region.c_str());
}
return EXIT_FAILURE;
@ -4316,7 +4316,7 @@ static int s3fs_check_service()
//
if(!isLoop && (responseCode == 400 || responseCode == 403) && S3fsCurl::GetSignatureType() == signature_type_t::V2_OR_V4){
// switch sigv2
S3FS_PRN_CRIT("Failed to connect by sigv4, so retry to connect by signature version 2. But you should to review url and endpoint option.");
S3FS_PRN_CRIT("Failed to connect by sigv4, so retry to connect by signature version 2. But you should to review url and region option.");
// retry to check with sigv2
isLoop = true;
@ -5154,9 +5154,9 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
S3fsCurl::SetSignatureType(signature_type_t::V4_ONLY);
return 0;
}
else if(is_prefix(arg, "endpoint=")){
endpoint = strchr(arg, '=') + sizeof(char);
is_specified_endpoint = true;
else if(is_prefix(arg, "region=") || is_prefix(arg, "endpoint=")){
region = strchr(arg, '=') + sizeof(char);
is_region_specified = true;
return 0;
}
else if(0 == strcmp(arg, "use_path_request_style")){

2
src/s3fs_global.cpp
View File

@ -33,7 +33,7 @@ bool noxmlns = false;
std::string program_name;
std::string service_path = "/";
std::string s3host = "https://s3.amazonaws.com";
std::string endpoint = "us-east-1";
std::string region = "us-east-1";
std::string cipher_suites;
std::string instance_name;

5
src/s3fs_help.cpp
View File

@ -305,8 +305,8 @@ static constexpr char help_string[] =
" If you do not use https, please specify the URL with the url\n"
" option.\n"
"\n"
" endpoint (default=\"us-east-1\")\n"
" - sets the endpoint to use on signature version 4\n"
" region (default=\"us-east-1\")\n"
" - sets the region to use on signature version 4\n"
" If this option is not specified, s3fs uses \"us-east-1\" region as\n"
" the default. If the s3fs could not connect to the region specified\n"
" by this option, s3fs could not run. But if you do not specify this\n"
@ -314,6 +314,7 @@ static constexpr char help_string[] =
" will retry to automatically connect to the other region. So s3fs\n"
" can know the correct region name, because s3fs can find it in an\n"
" error from the S3 server.\n"
" You can also specify the legacy -o endpoint which means the same thing.\n"
"\n"
" sigv2 (default is signature version 4 falling back to version 2)\n"
" - sets signing AWS requests by using only signature version 2\n"

2
test/integration-test-common.sh
View File

@ -317,7 +317,7 @@ function start_s3fs {
${TEST_BUCKET_MOUNT_POINT_1} \
-o use_path_request_style \
-o url="${S3_URL}" \
-o endpoint="${S3_ENDPOINT}" \
-o region="${S3_ENDPOINT}" \
-o use_xattr=1 \
-o enable_unsigned_payload \
${AUTH_OPT} \