youngkingdom/s3fs-fuse
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed bugs about a object larger than free disk space

Browse Source
This commit is contained in:
Takeshi Nakatani
2015-10-20 15:19:04 +00:00
parent 1b323a6252
commit 83d46ef8c6
6 changed files with 99 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
doc/man/s3fs.1
View File

@ -161,8 +161,8 @@ It is necessary to set this value depending on a CPU and a network band.
.TP
\fB\-o\fR multipart_size(default="10"(10MB))
number of one part size in multipart uploading request.
The default size is 10MB(10485760byte), this value is minimum size.
Specify number of MB and over 10(MB).
The default size is 10MB(10485760byte), minimum value is 5MB(5242880byte).
Specify number of MB and over 5(MB).
.TP
\fB\-o\fR ensure_diskfree(default the same as multipart_size value)
sets MB to ensure disk free space. This option means the threshold of free space size on disk which is used for the cache file by s3fs.

2
src/curl.cpp
View File

@ -1025,7 +1025,7 @@ string S3fsCurl::SetIAMRole(const char* role)
bool S3fsCurl::SetMultipartSize(off_t size)
{
size = size * 1024 * 1024;
if(size < MULTIPART_SIZE){
if(size < MIN_MULTIPART_SIZE){
return false;
}
S3fsCurl::multipart_size = size;

5
src/curl.h
View File

@ -20,6 +20,11 @@
#ifndef S3FS_CURL_H_
#define S3FS_CURL_H_
//----------------------------------------------
// Symbols
//----------------------------------------------
#define MIN_MULTIPART_SIZE 5242880 // 5MB
//----------------------------------------------
// class BodyData
//----------------------------------------------

134
src/fdcache.cpp
View File

@ -317,17 +317,15 @@ bool PageList::Resize(size_t size, bool is_loaded)
}else if(size < total){
// cut area
fdpage_list_t::iterator iter;
for(fdpage_list_t::reverse_iterator riter = pages.rbegin(); riter != pages.rend(); ){
if(size < static_cast<size_t>((*riter)->offset)){
// cut over area
iter = riter.base();
++riter;
pages.erase(iter);
}else{ // size < static_cast<size_t>((*riter)->offset + (*riter)->bytes)
// change size of last area
(*riter)->bytes = size - static_cast<size_t>((*riter)->offset);
break;
for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ){
if(static_cast<size_t>((*iter)->next()) <= size){
++iter;
}else{
if(size <= static_cast<size_t>((*iter)->offset)){
iter = pages.erase(iter);
}else{
(*iter)->bytes = size - static_cast<size_t>((*iter)->offset);
}
}
}
}else{ // total == size
@ -408,45 +406,62 @@ bool PageList::FindUnloadedPage(off_t start, off_t& resstart, size_t& ressize) c
size_t PageList::GetTotalUnloadedPageSize(off_t start, size_t size) const
{
size_t restsize = 0;
off_t next = static_cast<off_t>(start + size);
for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){
if(start <= (*iter)->end()){
if(0 != size && static_cast<size_t>(start + size) <= static_cast<size_t>((*iter)->offset)){
// reach to end
break;
if((*iter)->next() <= start){
continue;
}
if(next <= (*iter)->offset){
break;
}
if((*iter)->loaded){
continue;
}
size_t tmpsize;
if((*iter)->offset <= start){
if((*iter)->next() <= next){
tmpsize = static_cast<size_t>((*iter)->next() - start);
}else{
tmpsize = static_cast<size_t>(next - start); // = size
}
// after start pos
if(!(*iter)->loaded){
// found unloadedialized area
restsize += (*iter)->bytes;
}else{
if((*iter)->next() <= next){
tmpsize = static_cast<size_t>((*iter)->next() - (*iter)->offset); // = (*iter)->bytes
}else{
tmpsize = static_cast<size_t>(next - (*iter)->offset);
}
}
restsize += tmpsize;
}
return restsize;
}
int PageList::GetUnloadedPages(fdpage_list_t& unloaded_list, off_t start, size_t size) const
{
for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){
if(start < (*iter)->offset){
continue;
// If size is 0, it means loading to end.
if(0 == size){
if(static_cast<size_t>(start) < Size()){
size = static_cast<size_t>(Size() - start);
}
}
off_t next = static_cast<off_t>(start + size);
for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){
if((*iter)->next() <= start){
continue;
}
if(0 != size && static_cast<size_t>(start + size) <= static_cast<size_t>((*iter)->offset)){
break; // reach end
if(next <= (*iter)->offset){
break;
}
if((*iter)->loaded){
continue; // already loaded
}
// page area
off_t page_start = max((*iter)->offset, start);
size_t page_size;
if(0 == size || (*iter)->next() < static_cast<off_t>(start + size)){
page_size = static_cast<size_t>((*iter)->next() - page_start);
}else{
page_size = static_cast<size_t>((start + size) - page_start);
}
off_t page_next = min((*iter)->next(), next);
size_t page_size = static_cast<size_t>(page_next - page_start);
// add list
fdpage_list_t::reverse_iterator riter = unloaded_list.rbegin();
if(riter != unloaded_list.rend() && (*riter)->next() == page_start){
@ -1096,6 +1111,22 @@ int FdEntity::NoCacheLoadAndPost(off_t start, size_t size)
off_t offset = (*iter)->offset + totalread;
oneread = min(((*iter)->bytes - totalread), static_cast<size_t>(S3fsCurl::GetMultipartSize()));
// check rest size is over minimum part size
//
// [NOTE]
// If the final part size is smaller than 5MB, it is not allowed by S3 API.
// For this case, if the previous part of the final part is not over 5GB,
// we incorporate the final part to the previous part. If the previous part
// is over 5GB, we want to even out the last part and the previous part.
//
if(((*iter)->bytes - totalread - oneread) < MIN_MULTIPART_SIZE){
if(FIVE_GB < ((*iter)->bytes - totalread)){
oneread = ((*iter)->bytes - totalread) / 2;
}else{
oneread = ((*iter)->bytes - totalread);
}
}
if(!(*iter)->loaded){
//
// loading or initializing
@ -1405,21 +1436,23 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)
}
}
}
}
// load size(for prefetch)
size_t load_size = size;
if(static_cast<size_t>(start + size) < pagelist.Size()){
if(static_cast<size_t>(start + S3fsCurl::GetMultipartSize()) < pagelist.Size()){
load_size = S3fsCurl::GetMultipartSize();
}else{
load_size = static_cast<size_t>(pagelist.Size() - start);
}
}
// Loading
if(0 < size && 0 != (result = Load(start, load_size))){
S3FS_PRN_ERR("could not download. start(%jd), size(%zu), errno(%d)", (intmax_t)start, size, result);
return -EIO;
// load size(for prefetch)
size_t load_size = size;
if(static_cast<size_t>(start + size) < pagelist.Size()){
size_t prefetch_max_size = max(size, static_cast<size_t>(S3fsCurl::GetMultipartSize()));
if(static_cast<size_t>(start + prefetch_max_size) < pagelist.Size()){
load_size = prefetch_max_size;
}else{
load_size = static_cast<size_t>(pagelist.Size() - start);
}
}
// Loading
if(0 < size && 0 != (result = Load(start, load_size))){
S3FS_PRN_ERR("could not download. start(%jd), size(%zu), errno(%d)", (intmax_t)start, size, result);
return -EIO;
}
}
// Reading
if(-1 == (rsize = pread(fd, bytes, size, start))){
@ -1446,6 +1479,7 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
size_t restsize = pagelist.GetTotalUnloadedPageSize(0, start) + size;
if(FdManager::IsSafeDiskSpace(NULL, restsize)){
// enough disk space
// Load unitialized area which starts from 0 to (start + size) before writing.
if(0 < start && 0 != (result = Load(0, static_cast<size_t>(start)))){
S3FS_PRN_ERR("failed to load uninitialized area before writing(errno=%d)", result);
@ -1641,9 +1675,17 @@ size_t FdManager::SetEnsureFreeDiskSpace(size_t size)
{
size_t old = FdManager::free_disk_space;
if(0 == size){
FdManager::free_disk_space = static_cast<size_t>(S3fsCurl::GetMultipartSize());
if(0 == FdManager::free_disk_space){
FdManager::free_disk_space = static_cast<size_t>(S3fsCurl::GetMultipartSize());
}
}else{
FdManager::free_disk_space = max(size, static_cast<size_t>(S3fsCurl::GetMultipartSize()));
if(0 == FdManager::free_disk_space){
FdManager::free_disk_space = max(size, static_cast<size_t>(S3fsCurl::GetMultipartSize()));
}else{
if(static_cast<size_t>(S3fsCurl::GetMultipartSize()) <= size){
FdManager::free_disk_space = size;
}
}
}
return old;
}

1
src/fdcache.h
View File

@ -206,7 +206,6 @@ class FdManager
static size_t GetEnsureFreeDiskSpace(void) { return FdManager::free_disk_space; }
static size_t SetEnsureFreeDiskSpace(size_t size);
static size_t SetEnsureFreeDiskSpace(void) { return FdManager::SetEnsureFreeDiskSpace(FdManager::free_disk_space); }
static size_t InitEnsureFreeDiskSpace(void) { return FdManager::SetEnsureFreeDiskSpace(0); }
static bool IsSafeDiskSpace(const char* path, size_t size);

6
src/s3fs.cpp
View File

@ -4469,11 +4469,11 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
if(0 == STR2NCMP(arg, "multipart_size=")){
off_t size = static_cast<off_t>(s3fs_strtoofft(strchr(arg, '=') + sizeof(char)));
if(!S3fsCurl::SetMultipartSize(size)){
S3FS_PRN_EXIT("multipart_size option must be at least 10 MB.");
S3FS_PRN_EXIT("multipart_size option must be at least 5 MB.");
return -1;
}
// update ensure free disk space
FdManager::SetEnsureFreeDiskSpace();
// update ensure free disk space if it is not set.
FdManager::InitEnsureFreeDiskSpace();
return 0;
}
if(0 == STR2NCMP(arg, "ensure_diskfree=")){