diff --git a/doc/man/s3fs.1 b/doc/man/s3fs.1 index ba11c31..2bb06d9 100644 --- a/doc/man/s3fs.1 +++ b/doc/man/s3fs.1 @@ -161,8 +161,8 @@ It is necessary to set this value depending on a CPU and a network band. .TP \fB\-o\fR multipart_size(default="10"(10MB)) number of one part size in multipart uploading request. -The default size is 10MB(10485760byte), this value is minimum size. -Specify number of MB and over 10(MB). +The default size is 10MB(10485760byte), minimum value is 5MB(5242880byte). +Specify number of MB and over 5(MB). .TP \fB\-o\fR ensure_diskfree(default the same as multipart_size value) sets MB to ensure disk free space. This option means the threshold of free space size on disk which is used for the cache file by s3fs. diff --git a/src/curl.cpp b/src/curl.cpp index 4d5c453..d0bb01d 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -1025,7 +1025,7 @@ string S3fsCurl::SetIAMRole(const char* role) bool S3fsCurl::SetMultipartSize(off_t size) { size = size * 1024 * 1024; - if(size < MULTIPART_SIZE){ + if(size < MIN_MULTIPART_SIZE){ return false; } S3fsCurl::multipart_size = size; diff --git a/src/curl.h b/src/curl.h index 79251cc..0da7ccb 100644 --- a/src/curl.h +++ b/src/curl.h @@ -20,6 +20,11 @@ #ifndef S3FS_CURL_H_ #define S3FS_CURL_H_ +//---------------------------------------------- +// Symbols +//---------------------------------------------- +#define MIN_MULTIPART_SIZE 5242880 // 5MB + //---------------------------------------------- // class BodyData //---------------------------------------------- diff --git a/src/fdcache.cpp b/src/fdcache.cpp index ec09035..bc08122 100644 --- a/src/fdcache.cpp +++ b/src/fdcache.cpp @@ -317,17 +317,15 @@ bool PageList::Resize(size_t size, bool is_loaded) }else if(size < total){ // cut area - fdpage_list_t::iterator iter; - for(fdpage_list_t::reverse_iterator riter = pages.rbegin(); riter != pages.rend(); ){ - if(size < static_cast((*riter)->offset)){ - // cut over area - iter = riter.base(); - ++riter; - pages.erase(iter); - }else{ // size < static_cast((*riter)->offset + (*riter)->bytes) - // change size of last area - (*riter)->bytes = size - static_cast((*riter)->offset); - break; + for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ){ + if(static_cast((*iter)->next()) <= size){ + ++iter; + }else{ + if(size <= static_cast((*iter)->offset)){ + iter = pages.erase(iter); + }else{ + (*iter)->bytes = size - static_cast((*iter)->offset); + } } } }else{ // total == size @@ -408,45 +406,62 @@ bool PageList::FindUnloadedPage(off_t start, off_t& resstart, size_t& ressize) c size_t PageList::GetTotalUnloadedPageSize(off_t start, size_t size) const { size_t restsize = 0; + off_t next = static_cast(start + size); for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){ - if(start <= (*iter)->end()){ - if(0 != size && static_cast(start + size) <= static_cast((*iter)->offset)){ - // reach to end - break; + if((*iter)->next() <= start){ + continue; + } + if(next <= (*iter)->offset){ + break; + } + if((*iter)->loaded){ + continue; + } + size_t tmpsize; + if((*iter)->offset <= start){ + if((*iter)->next() <= next){ + tmpsize = static_cast((*iter)->next() - start); + }else{ + tmpsize = static_cast(next - start); // = size } - // after start pos - if(!(*iter)->loaded){ - // found unloadedialized area - restsize += (*iter)->bytes; + }else{ + if((*iter)->next() <= next){ + tmpsize = static_cast((*iter)->next() - (*iter)->offset); // = (*iter)->bytes + }else{ + tmpsize = static_cast(next - (*iter)->offset); } } + restsize += tmpsize; } return restsize; } int PageList::GetUnloadedPages(fdpage_list_t& unloaded_list, off_t start, size_t size) const { - for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){ - if(start < (*iter)->offset){ - continue; + // If size is 0, it means loading to end. + if(0 == size){ + if(static_cast(start) < Size()){ + size = static_cast(Size() - start); } + } + off_t next = static_cast(start + size); + + for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){ if((*iter)->next() <= start){ continue; } - if(0 != size && static_cast(start + size) <= static_cast((*iter)->offset)){ - break; // reach end + if(next <= (*iter)->offset){ + break; } if((*iter)->loaded){ continue; // already loaded } + // page area off_t page_start = max((*iter)->offset, start); - size_t page_size; - if(0 == size || (*iter)->next() < static_cast(start + size)){ - page_size = static_cast((*iter)->next() - page_start); - }else{ - page_size = static_cast((start + size) - page_start); - } + off_t page_next = min((*iter)->next(), next); + size_t page_size = static_cast(page_next - page_start); + // add list fdpage_list_t::reverse_iterator riter = unloaded_list.rbegin(); if(riter != unloaded_list.rend() && (*riter)->next() == page_start){ @@ -1096,6 +1111,22 @@ int FdEntity::NoCacheLoadAndPost(off_t start, size_t size) off_t offset = (*iter)->offset + totalread; oneread = min(((*iter)->bytes - totalread), static_cast(S3fsCurl::GetMultipartSize())); + // check rest size is over minimum part size + // + // [NOTE] + // If the final part size is smaller than 5MB, it is not allowed by S3 API. + // For this case, if the previous part of the final part is not over 5GB, + // we incorporate the final part to the previous part. If the previous part + // is over 5GB, we want to even out the last part and the previous part. + // + if(((*iter)->bytes - totalread - oneread) < MIN_MULTIPART_SIZE){ + if(FIVE_GB < ((*iter)->bytes - totalread)){ + oneread = ((*iter)->bytes - totalread) / 2; + }else{ + oneread = ((*iter)->bytes - totalread); + } + } + if(!(*iter)->loaded){ // // loading or initializing @@ -1405,21 +1436,23 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load) } } } - } - // load size(for prefetch) - size_t load_size = size; - if(static_cast(start + size) < pagelist.Size()){ - if(static_cast(start + S3fsCurl::GetMultipartSize()) < pagelist.Size()){ - load_size = S3fsCurl::GetMultipartSize(); - }else{ - load_size = static_cast(pagelist.Size() - start); - } - } - // Loading - if(0 < size && 0 != (result = Load(start, load_size))){ - S3FS_PRN_ERR("could not download. start(%jd), size(%zu), errno(%d)", (intmax_t)start, size, result); - return -EIO; + // load size(for prefetch) + size_t load_size = size; + if(static_cast(start + size) < pagelist.Size()){ + size_t prefetch_max_size = max(size, static_cast(S3fsCurl::GetMultipartSize())); + + if(static_cast(start + prefetch_max_size) < pagelist.Size()){ + load_size = prefetch_max_size; + }else{ + load_size = static_cast(pagelist.Size() - start); + } + } + // Loading + if(0 < size && 0 != (result = Load(start, load_size))){ + S3FS_PRN_ERR("could not download. start(%jd), size(%zu), errno(%d)", (intmax_t)start, size, result); + return -EIO; + } } // Reading if(-1 == (rsize = pread(fd, bytes, size, start))){ @@ -1446,6 +1479,7 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size) size_t restsize = pagelist.GetTotalUnloadedPageSize(0, start) + size; if(FdManager::IsSafeDiskSpace(NULL, restsize)){ // enough disk space + // Load unitialized area which starts from 0 to (start + size) before writing. if(0 < start && 0 != (result = Load(0, static_cast(start)))){ S3FS_PRN_ERR("failed to load uninitialized area before writing(errno=%d)", result); @@ -1641,9 +1675,17 @@ size_t FdManager::SetEnsureFreeDiskSpace(size_t size) { size_t old = FdManager::free_disk_space; if(0 == size){ - FdManager::free_disk_space = static_cast(S3fsCurl::GetMultipartSize()); + if(0 == FdManager::free_disk_space){ + FdManager::free_disk_space = static_cast(S3fsCurl::GetMultipartSize()); + } }else{ - FdManager::free_disk_space = max(size, static_cast(S3fsCurl::GetMultipartSize())); + if(0 == FdManager::free_disk_space){ + FdManager::free_disk_space = max(size, static_cast(S3fsCurl::GetMultipartSize())); + }else{ + if(static_cast(S3fsCurl::GetMultipartSize()) <= size){ + FdManager::free_disk_space = size; + } + } } return old; } diff --git a/src/fdcache.h b/src/fdcache.h index aca5d1d..2ca9563 100644 --- a/src/fdcache.h +++ b/src/fdcache.h @@ -206,7 +206,6 @@ class FdManager static size_t GetEnsureFreeDiskSpace(void) { return FdManager::free_disk_space; } static size_t SetEnsureFreeDiskSpace(size_t size); - static size_t SetEnsureFreeDiskSpace(void) { return FdManager::SetEnsureFreeDiskSpace(FdManager::free_disk_space); } static size_t InitEnsureFreeDiskSpace(void) { return FdManager::SetEnsureFreeDiskSpace(0); } static bool IsSafeDiskSpace(const char* path, size_t size); diff --git a/src/s3fs.cpp b/src/s3fs.cpp index a6e77bf..52cdbb1 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -4469,11 +4469,11 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar if(0 == STR2NCMP(arg, "multipart_size=")){ off_t size = static_cast(s3fs_strtoofft(strchr(arg, '=') + sizeof(char))); if(!S3fsCurl::SetMultipartSize(size)){ - S3FS_PRN_EXIT("multipart_size option must be at least 10 MB."); + S3FS_PRN_EXIT("multipart_size option must be at least 5 MB."); return -1; } - // update ensure free disk space - FdManager::SetEnsureFreeDiskSpace(); + // update ensure free disk space if it is not set. + FdManager::InitEnsureFreeDiskSpace(); return 0; } if(0 == STR2NCMP(arg, "ensure_diskfree=")){