Merge pull request #978 from ggtakec/master

Updated ChangeLog and configure.ac etc for release 1.85

.clang-tidy

@@ -0,0 +1,26 @@
+Checks: '
+    -*,
+    bugprone-*,
+    google-*,
+    -google-build-using-namespace,
+    -google-readability-casting,
+    -google-readability-todo,
+    -google-runtime-int,
+    -google-runtime-references,
+    misc-*,
+    -misc-misplaced-const,
+    -misc-redundant-expression,
+    -misc-unused-parameters,
+    modernize-*,
+    -modernize-deprecated-headers,
+    -modernize-loop-convert,
+    -modernize-use-auto,
+    -modernize-use-nullptr,
+    -modernize-use-using,
+    performance-*,
+    portability-*,
+    readability-*,
+    -readability-else-after-return,
+    -readability-implicit-bool-conversion,
+    -readability-named-parameter,
+    -readability-simplify-boolean-expr'

.gitattributes

@@ -0,0 +1,32 @@
+#
+# s3fs - FUSE-based file system backed by Amazon S3
+#
+# Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+
+* text eol=lf
+
+*.png   binary
+
+#
+# Local variables:
+# tab-width: 4
+# c-basic-offset: 4
+# End:
+# vim600: noet sw=4 ts=4 fdm=marker
+# vim<600: noet sw=4 ts=4
+#

.github/ISSUE_TEMPLATE.md

@@ -1,27 +1,28 @@
-#### Additional Information
+### Additional Information
 _The following information is very important in order to help us to help you. Omission of the following details may delay your support request or receive no attention at all._
+_Keep in mind that the commands we provide to retrieve information are oriented to GNU/Linux Distributions, so you could need to use others if you use s3fs on macOS or BSD_
 
-- Version of s3fs being used (s3fs --version)
- - _example: 1.0_
+#### Version of s3fs being used (s3fs --version)
+_example: 1.00_
 
-- Version of fuse being used (pkg-config --modversion fuse)
- - _example: 2.9.4_
+#### Version of fuse being used (pkg-config --modversion fuse, rpm -qi fuse, dpkg -s fuse)
+_example: 2.9.4_
 
-- System information (uname -a)
- - _command result: uname -a_
+#### Kernel information (uname -r)
+_command result: uname -r_
 
-- Distro (cat /etc/issue)
- - _command result: result_
+#### GNU/Linux Distribution, if applicable (cat /etc/os-release)
+_command result: cat /etc/os-release_
 
-- s3fs command line used (if applicable)
+#### s3fs command line used, if applicable
 ```
 ```
-- /etc/fstab entry (if applicable): 
+#### /etc/fstab entry, if applicable
 ```
 ```
-- s3fs syslog messages (grep s3fs /var/log/syslog, or s3fs outputs)  
+#### s3fs syslog messages (grep s3fs /var/log/syslog, journalctl | grep s3fs, or s3fs outputs)
 _if you execute s3fs with dbglevel, curldbg option, you can get detail debug messages_
 ```
 ```
-#### Details about issue
+### Details about issue
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,5 @@
-#### Relevant Issue (if applicable)
+### Relevant Issue (if applicable)
 _If there are Issues related to this PullRequest, please list it._
 
-#### Details
+### Details
 _Please describe the details of PullRequest._

.gitignore

@@ -1,31 +1,86 @@
+#
+# s3fs - FUSE-based file system backed by Amazon S3
+#
+# Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+
+#
+# Compiled Object files
+#
+*.slo
+*.lo
 *.o
-/Makefile
-/Makefile.in
-/aclocal.m4
-/autom4te.cache/
-/config.guess
-/config.log
-/config.status
-/config.sub
-/stamp-h1
-/config.h
-/config.h.in
-/config.h.in~
-/configure
-/depcomp
-/test-driver
-/compile
-/doc/Makefile
-/doc/Makefile.in
-/install-sh
-/missing
-/src/.deps/
-/src/Makefile
-/src/Makefile.in
-/src/s3fs
-/src/test_*
-/test/.deps/
-/test/Makefile
-/test/Makefile.in
-/test/*.log
-/default_commit_hash
+*.Po
+*.Plo
+
+#
+# autotools/automake
+#
+aclocal.m4
+autom4te.cache
+autoscan.log
+config.guess
+config.h
+config.h.in
+config.h.in~
+config.log
+config.status
+config.sub
+configure
+configure.scan
+depcomp
+install-sh
+libtool
+ltmain.sh
+m4
+m4/*
+missing
+stamp-h1
+Makefile
+Makefile.in
+test-driver
+compile
+missing
+
+#
+# object directories
+#
+.deps
+.libs
+*/.deps
+*/.deps/*
+*/.libs
+*/.libs/*
+
+#
+# each directories
+#
+*.log
+*.trs
+default_commit_hash
+src/s3fs
+src/test_*
+test/s3proxy-*
+
+#
+# Local variables:
+# tab-width: 4
+# c-basic-offset: 4
+# End:
+# vim600: noet sw=4 ts=4 fdm=marker
+# vim<600: noet sw=4 ts=4
+#

.travis.yml

@@ -1,17 +1,98 @@
+#
+# s3fs - FUSE-based file system backed by Amazon S3
+#
+# Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+
 language: cpp
-sudo: required
-dist: trusty
-cache: apt
-before_install:
- - sudo apt-get update -qq
- - sudo apt-get install -qq cppcheck libfuse-dev openjdk-7-jdk
- - sudo update-alternatives --set java /usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java
-script:
- - ./autogen.sh
- - ./configure
- - make
- - make cppcheck
- - make check -C src
- - modprobe fuse
- - make check -C test
- - cat test/test-suite.log
+
+matrix:
+  include:
+    - os: linux
+      sudo: required
+      dist: trusty
+      cache: apt
+      before_install:
+        - sudo apt-get update -qq
+        - sudo apt-get install -qq attr cppcheck libfuse-dev openjdk-7-jdk
+        - sudo update-alternatives --set java /usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java
+      script:
+        - ./autogen.sh
+        - ./configure CPPFLAGS='-I/usr/local/opt/openssl/include' CXXFLAGS='-std=c++03'
+        - make
+        - make cppcheck
+        - make check -C src
+        - modprobe fuse
+        - make check -C test
+        - cat test/test-suite.log
+
+    - os: osx
+      osx_image: xcode8.3
+      before_install:
+        - brew update
+        - brew install truncate
+        - brew tap caskroom/cask
+        - brew cask install osxfuse
+        - if [ -f /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ]; then sudo chmod +s /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ; elif [ -f /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ]; then sudo chmod +s /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ; fi
+        - brew install gnu-sed
+        - sudo ln -s /usr/local/opt/gnu-sed/bin/gsed /usr/local/bin/sed
+        - sudo ln -s /usr/local/opt/coreutils/bin/gstdbuf /usr/local/bin/stdbuf
+        - brew install cppcheck
+      script:
+        - ./autogen.sh
+        - PKG_CONFIG_PATH=/usr/local/opt/curl/lib/pkgconfig:/usr/local/opt/openssl/lib/pkgconfig ./configure CXXFLAGS='-std=c++03'
+        - make
+        - make cppcheck
+        - make check -C src
+        - if [ -f /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ]; then /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ; elif [ -f /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ]; then /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ; fi
+        - make check -C test
+        - cat test/test-suite.log
+
+
+    - os: linux-ppc64le
+      sudo: required
+      dist: trusty
+      cache: apt
+      before_install:
+        - sudo add-apt-repository -y ppa:openjdk-r/ppa 
+        - sudo apt-get update -qq
+        - sudo apt-get install -qq attr libfuse-dev openjdk-7-jdk
+        - sudo update-alternatives --set java /usr/lib/jvm/java-7-openjdk-ppc64el/jre/bin/java
+        - sudo git clone --branch 1.61 https://github.com/danmar/cppcheck.git
+        - pwd
+        - cd ./cppcheck
+        - sudo make
+        - sudo make install
+        - cd ../ 
+      script:
+        - ./autogen.sh
+        - ./configure CPPFLAGS='-I/usr/local/opt/openssl/include' CXXFLAGS='-std=c++03'
+        - make
+        - make cppcheck
+        - make check -C src
+        - modprobe fuse
+        - make check -C test
+        - cat test/test-suite.log
+
+#
+# Local variables:
+# tab-width: 4
+# c-basic-offset: 4
+# End:
+# vim600: noet sw=4 ts=4 fdm=marker
+# vim<600: noet sw=4 ts=4
+#

AUTHORS

@@ -17,3 +17,7 @@ Bugfixes, performance and other improvements.
 5. Takeshi Nakatani <ggtakec@gmail.com>
 
 Bugfixes, performance and other improvements.
+
+6. Andrew Gaul <gaul@gaul.org>
+
+Bugfixes, performance and other improvements.

ChangeLog

@@ -1,8 +1,172 @@
-ChangeLog for S3FS
+ChangeLog for S3FS
 ------------------
 
+Version 1.85 -- 11 Mar, 2019
+#804 - add Backblaze B2
+#812 - Fix typo s/mutliple/multiple/
+#819 - #691: Made instructions for creating password file more obvious.
+#820 - Enable big writes if capable
+#826 - For RPM distributions fuse-libs is enough
+#831 - Add support for storage class ONEZONE_IA.
+#832 - Simplify hex conversion
+#833 - New installation instructions for Fedora >= 27 and CentOS7
+#834 - Improve template for issues
+#835 - Make the compilation instructions generic
+#840 - Replace all mentions to MacOS X to macOS
+#849 - Correct typo
+#851 - Correctly compare list_object_max_keys
+#852 - Allow credentials from ${HOME}/.aws/credentials
+#853 - Replace ~ with ${HOME} in examples
+#855 - Include StackOverflow in FAQs
+#856 - Add icon for s3fs
+#859 - Upload S3 parts without batching
+#861 - Add 'profile' option to command line help.
+#865 - fix multihead warning check
+#866 - Multi-arch support for ppc64le
+#870 - Correct typos in command-line parsing
+#874 - Address cppcheck 1.86 errors
+#877 - Check arguments and environment before .aws/creds
+#882 - [curl] Assume long encryption keys are base64 encoded
+#885 - Update s3fs_util.cpp for correspondence of Nextcloud contype
+#888 - Add Server Fault to FAQs
+#892 - Repair xattr tests
+#893 - Store and retrieve file change time
+#894 - Default uid/gid/mode when object lacks permissions
+#895 - Emit more friendly error for buckets with dots
+#898 - Flush file before renaming
+#899 - Tighten up HTTP response code check
+#900 - Plug memory leak
+#901 - Plug memory leaks
+#902 - Avoid pass-by-value when not necessary
+#903 - Prefer find(char) over find(const char *)
+#904 - Remove unnecessary calls to std::string::c_str
+#905 - Fix comparison in s3fs_strtoofft
+#906 - Prefer HTTPS links where possible
+#908 - Added an error message when HTTP 301 status
+#909 - Ignore after period character of floating point in x-amz-meta-mtime
+#910 - Added a missing extension to .gitignore, and formatted dot files
+#911 - Added detail error message when HTTP 301/307 status
+#912 - Automatic region change made possible other than us-east-1(default)
+#913 - Prefer abort over assert(false)
+#914 - Issue readdir HEAD requests without batching
+#917 - Reference better-known AWS CLI for compatibility
+#918 - Load tail range during overwrite
+#919 - Add test for mv non-empty directory
+#920 - Remove unnecessary string copies
+#921 - Remove redundant string initializations
+#923 - Reverted automatic region change and changed messages
+#924 - Prefer empty over size checks
+#925 - Remove redundant null checks before delete
+#926 - Accept paths with : in them
+#930 - Correct enable_content_md5 docs
+#931 - Correct sigv2 typo
+#932 - Prefer AutoLock for synchronization
+#933 - Remove mirror path when deleting cache
+#934 - Checked and corrected all typo
+#937 - Disable malloc_trim
+#938 - Remove unneeded void parameter
+#939 - Prefer specific [io]stringstream where possible
+#940 - Copy parts in parallel
+#942 - Ensure s3fs compiles with C++03
+#943 - Return not supported when hard linking
+#944 - Repair utility mode
+#946 - Simplify async request completion code
+#948 - Add logging for too many parts
+#949 - Implement exponential backoff for 503
+#950 - Added S3FS_MALLOC_TRIM build switch
+#951 - Added a non-interactive option to utility mode
+#952 - Automatically abort failed multipart requests
+#953 - Update s3ql link
+#954 - Clear containers instead of individual erases
+#955 - Address miscellaneous clang-tidy warnings
+#957 - Upgrade to S3Proxy 1.6.1
+#958 - Document lack of inotify support
+#959 - Fixed code for latest cppcheck error on OSX
+#960 - Wtf8
+#961 - Work around cppcheck warnings
+#965 - Improvement of curl session pool for multipart
+#967 - Increase FdEntity reference count when returning
+#969 - Fix lazy typo
+#970 - Remove from file from stat cache during rename
+#972 - Add instructions for Amazon Linux
+#974 - Changed the description order of man page options
+#975 - Fixed ref-count when error occurred.
+#977 - Make macOS instructions consistent with others
+
+Version 1.84 -- Jul 8, 2018
+#704 - Update README.md with details about .passwd-s3fs
+#710 - add disk space reservation
+#712 - Added Cygwin build options
+#714 - reduce lock contention on file open
+#724 - don't fail multirequest on single thread error
+#726 - add an instance_name option for logging
+#727 - Fixed Travis CI error about cppcheck - #713
+#729 - FreeBSD build fixes
+#733 - More useful error message for dupe entries in passwd file
+#739 - cleanup curl handle state on retries
+#745 - don't fail mkdir when directory exists
+#753 - fix xpath selector in bucket listing
+#754 - Validate the URL format for http/https
+#755 - Added reset curl handle when returning to handle pool
+#756 - Optimize defaults
+#761 - Simplify installation for Ubuntu 16.04
+#762 - Upgrade to S3Proxy 1.6.0
+#763 - cleanup curl handles before curl share
+#764 - Remove false multihead warnings
+#765 - Add Debian installation instructions
+#766 - Remove s3fs-python
+#768 - Fixed memory leak
+#769 - Revert "enable FUSE read_sync by default"
+#774 - Option for IAM authentication endpoint
+#780 - gnutls_auth: initialize libgcrypt
+#781 - Fixed an error by cppcheck on OSX
+#786 - Log messages for 5xx and 4xx HTTP response code
+#789 - Instructions for SUSE and openSUSE prebuilt packages
+#793 - Added list_object_max_keys option based on #783 PR
+
+Version 1.83 -- Dec 17, 2017
+#606 - Add Homebrew instructions
+#608 - Fix chown_nocopy losing existing uid/gid if unspecified
+#609 - Group permission checks sometimes fail with large number of groups
+#611 - Fixed clock_gettime build failure on macOS 10.12 Sierra - #600
+#621 - Upgrade to S3Proxy 1.5.3
+#627 - Update README.md
+#630 - Added travis test on osx for #601
+#631 - Merged macosx branch into master branch #601
+#636 - Fix intermittent upload failures on macOS
+#637 - Add blurb about non-Amazon S3 implementations
+#638 - Minor fixes to README
+#639 - Update Homebrew instructions
+#642 - Fixed potential atomic violation in S3fsCurl::AddUserAgent - #633
+#644 - Fixed with unnecessary equal in POST uploads url argument - #643
+#645 - Configure S3Proxy for SSL
+#646 - Simplify S3Proxy PID handling
+#652 - Fix s3fs_init message
+#659 - Do not fail updating directory when removing old-style object(ref #658)
+#660 - Refixed s3fs_init message(ref #652)
+#663 - Lock FdEntity when mutating orgmeta
+#664 - auth headers insertion refactoring
+#668 - Changed .travis.yml for fixing not found gpg2 on osx
+#669 - add IBM IAM authentication support
+#670 - Fixed a bug in S3fsCurl::LocateBundle
+#671 - Add support for ECS metadata endpoint
+#675 - Reduce use of preprocessor
+#676 - Move str definition from header to implementation
+#677 - Add s3proxy to .gitignore
+#679 - README.md Addition
+#681 - Changed functions about reading passwd file
+#684 - Correct signedness warning
+#686 - remove use of jsoncpp
+#688 - Improved use of temporary files - #678
+#690 - Added option ecs description to man page
+#692 - Updated template md files for issue and pr
+#695 - fix condition for parallel download
+#697 - Fixing race condition in FdEntity::GetStats
+#699 - Fix dbglevel usage
+
 Version 1.82 -- May 13, 2017
 #597 - Not fallback to HTTP - #596
+#598 - Updated ChangeLog and configure.ac for release 1.82
 
 Version 1.81 -- May 13, 2017
 #426 - Updated to correct ChangeLog
@@ -40,7 +204,7 @@ Version 1.81 -- May 13, 2017
 #540 - Address cppcheck 1.77 warnings
 #545 - Changed base cached time of stat_cache_expire option - #523
 #546 - Fixed double initialization of SSL library at foreground
-#550 - Add umount instruction for unplivileged user
+#550 - Add umount instruction for unprivileged user
 #551 - Updated stat_cache_expire option description - #545
 #552 - switch S3fsMultiCurl to use foreground threads
 #553 - add TLS cipher suites customization
@@ -93,7 +257,7 @@ Version 1.80 -- May 29, 2016
 #250 - s3fs can print version with short commit hash - #228
 #251 - Skip xattr tests if utilities are missing
 #252 - This fixes an issue with caching when the creation of a subdirectory …
-#253 - Added chacking cache dir perms at starting.
+#253 - Added checking cache dir perms at starting.
 #256 - Add no atomic rename to limitations
 #257 - Update README.md: Bugfix password file permissions errors
 #258 - Update README.md to better explain mount upon boot
@@ -121,7 +285,7 @@ Version 1.80 -- May 29, 2016
 #306 - Fix read concurrency to work in parallel count
 #307 - Fix pthread portability problem
 #308 - Changed ensure free disk space as additional change for #306
-#309 - Check pthread prtability in configure as additional change for #307
+#309 - Check pthread portability in configure as additional change for #307
 #310 - Update integration-test-main.sh as additional change for #300
 #311 - Change error log to debug log in s3fs_read()
 #313 - fix gitignore
@@ -133,14 +297,14 @@ Version 1.80 -- May 29, 2016
 #330 - Pass by const reference where possible
 #331 - Address various clang warnings
 #334 - Bucket host should include port and not path
-#336 - update REAME.md for fstab
+#336 - update README.md for fstab
 #338 - Fixed a bug about IAMCRED type could not be retried.
 #339 - Updated README.md for fstab example.
 #341 - Fix the memory leak issue in fdcache.
 #346 - Fix empty directory check against AWS S3
 #348 - Integration test summary, continue on error
 #350 - Changed cache out logic for stat - #340
-#351 - Check cache dirctory path and attributes - #347
+#351 - Check cache directory path and attributes - #347
 #352 - Remove stat file cache dir if specified del_cache - #337
 #354 - Supported regex type for additional header format - #343
 #355 - Fixed codes about clock_gettime for osx
@@ -217,7 +381,7 @@ issue #184 - Add usage information for multipart_size
 issue #185 - Correct obvious typos in usage and README
 issue #190 - Add a no_check_certificate option.
 issue #194 - Tilda in a file-name breaks things (EPERM)
-issue #198 - Disasble integration tests for Travis 
+issue #198 - Disable integration tests for Travis 
 issue #199 - Supported extended attributes(retry)
 issue #200 - fixed fallback to sigv2 for bucket create and GCS
 issue #202 - Specialize {set,get}xattr for OS X

INSTALL

@@ -124,7 +124,7 @@ architecture at a time in the source code directory.  After you have
 installed the package for one architecture, use `make distclean' before
 reconfiguring for another architecture.
 
-   On MacOS X 10.5 and later systems, you can create libraries and
+   On macOS 10.5 and later systems, you can create libraries and
 executables that work on multiple system types--known as "fat" or
 "universal" binaries--by specifying multiple `-arch' options to the
 compiler but only a single `-arch' option to the preprocessor.  Like

Makefile.am

@@ -32,10 +32,14 @@ cppcheck:
 	cppcheck --quiet --error-exitcode=1 \
             --inline-suppr \
             --std=c++03 \
+            --xml \
+            -D HAVE_ATTR_XATTR_H \
+            -D HAVE_SYS_EXTATTR_H \
+            -D HAVE_MALLOC_TRIM \
             -U CURLE_PEER_FAILED_VERIFICATION \
             -U P_tmpdir \
-            --enable=all \
+            -U ENOATTR \
+            --enable=warning,style,information,missingInclude \
             --suppress=missingIncludeSystem \
-            --suppress=unusedFunction \
-            --suppress=variableScope \
+            --suppress=unmatchedSuppression \
             src/ test/

README.md

@@ -1,8 +1,9 @@
 s3fs
 ====
 
-s3fs allows Linux and Mac OS X to mount an S3 bucket via FUSE.
-s3fs preserves the native object format for files, allowing use of other tools like [s3cmd](http://s3tools.org/s3cmd).  
+s3fs allows Linux and macOS to mount an S3 bucket via FUSE.
+s3fs preserves the native object format for files, allowing use of other
+tools like [AWS CLI](https://github.com/aws/aws-cli).
 [![Build Status](https://travis-ci.org/s3fs-fuse/s3fs-fuse.svg?branch=master)](https://travis-ci.org/s3fs-fuse/s3fs-fuse)
 
 Features
@@ -19,75 +20,143 @@ Features
 * user-specified regions, including Amazon GovCloud
 * authenticate via v2 or v4 signatures
 
-Installation
-------------
+Installation from pre-built packages
+------------------------------------
 
-Ensure you have all the dependencies:
+Some systems provide pre-built packages:
 
-On Ubuntu 14.04:
+* On Debian 9 and Ubuntu 16.04 or newer:
 
-```
-sudo apt-get install automake autotools-dev g++ git libcurl4-gnutls-dev libfuse-dev libssl-dev libxml2-dev make pkg-config
-```
+  ```
+  sudo apt-get install s3fs
+  ```
 
-On CentOS 7:
+* On SUSE 12 or newer and openSUSE 42.1 or newer:
 
-```
-sudo yum install automake fuse fuse-devel gcc-c++ git libcurl-devel libxml2-devel make openssl-devel
-```
+  ```
+  sudo zypper in s3fs
+  ```
 
-Compile from master via the following commands:
+* On Fedora 27 and newer:
+  ```
+  sudo yum install s3fs-fuse
+  ```
 
-```
-git clone https://github.com/s3fs-fuse/s3fs-fuse.git
-cd s3fs-fuse
-./autogen.sh
-./configure
-make
-sudo make install
-```
+* On RHEL/CentOS 7 and newer through EPEL repositories:
+  ```
+  sudo yum install epel-release
+  sudo yum install s3fs-fuse
+  ```
+
+* On Amazon Linux through EPEL repositories:
+
+  ```
+  sudo amazon-linux-extras install epel
+  sudo yum install s3fs-fuse
+  ```
+
+* On macOS, install via [Homebrew](https://brew.sh/):
+
+  ```
+  brew cask install osxfuse
+  brew install s3fs
+  ```
+
+Compilation and installation from sources
+-----------------------------------------
+
+These are generic instructions to compile from the master branch, and should work on almost any GNU/Linux, macOS, BSD, or similar.
+
+If you want specific instructions for some distributions, check the [wiki](https://github.com/s3fs-fuse/s3fs-fuse/wiki/Installation-Notes).
+
+Keep in mind using the pre-built packages when available.
+
+1. Ensure your system satisfies build and runtime dependencies for:
+
+   * fuse >= 2.8.4
+   * automake
+   * gcc-c++
+   * make
+   * libcurl
+   * libxml2
+   * openssl
+
+2. Then compile from master via the following commands:
+
+   ```
+   git clone https://github.com/s3fs-fuse/s3fs-fuse.git
+   cd s3fs-fuse
+   ./autogen.sh
+   ./configure
+   make
+   sudo make install
+   ```
 
 Examples
 --------
 
-Enter your S3 identity and credential in a file `/path/to/passwd`:
+s3fs supports the standard
+[AWS credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html)
+stored in `${HOME}/.aws/credentials`.  Alternatively, s3fs supports a custom passwd file.
+
+The default location for the s3fs password file can be created:
+
+* using a .passwd-s3fs file in the users home directory (i.e. ${HOME}/.passwd-s3fs)
+* using the system-wide /etc/passwd-s3fs file
+
+Enter your credentials in a file `${HOME}/.passwd-s3fs` and set
+owner-only permissions:
 
 ```
-echo MYIDENTITY:MYCREDENTIAL > /path/to/passwd
-```
-
-Make sure the file has proper permissions (if you get 'permissions' error when mounting) `/path/to/passwd`:
-
-```
-chmod 600 /path/to/passwd
+echo ACCESS_KEY_ID:SECRET_ACCESS_KEY > ${HOME}/.passwd-s3fs
+chmod 600 ${HOME}/.passwd-s3fs
 ```
 
 Run s3fs with an existing bucket `mybucket` and directory `/path/to/mountpoint`:
 
 ```
-s3fs mybucket /path/to/mountpoint -o passwd_file=/path/to/passwd
+s3fs mybucket /path/to/mountpoint -o passwd_file=${HOME}/.passwd-s3fs
 ```
 
 If you encounter any errors, enable debug output:
 
 ```
-s3fs mybucket /path/to/mountpoint -o passwd_file=/path/to/passwd -d -d -f -o f2 -o curldbg
+s3fs mybucket /path/to/mountpoint -o passwd_file=${HOME}/.passwd-s3fs -o dbglevel=info -f -o curldbg
 ```
 
 You can also mount on boot by entering the following line to `/etc/fstab`:
 
 ```
 s3fs#mybucket /path/to/mountpoint fuse _netdev,allow_other 0 0
+```
 
 or
 
+```
 mybucket /path/to/mountpoint fuse.s3fs _netdev,allow_other 0 0
 ```
 
+If you use s3fs with a non-Amazon S3 implementation, specify the URL and path-style requests:
+
+```
+s3fs mybucket /path/to/mountpoint -o passwd_file=${HOME}/.passwd-s3fs -o url=https://url.to.s3/ -o use_path_request_style
+```
+
+or(fstab)
+```
+s3fs#mybucket /path/to/mountpoint fuse _netdev,allow_other,use_path_request_style,url=https://url.to.s3/ 0 0
+```
+
+To use IBM IAM Authentication, use the `-o ibm_iam_auth` option, and specify the Service Instance ID and API Key in your credentials file:
+```
+echo SERVICEINSTANCEID:APIKEY > /path/to/passwd
+```
+The Service Instance ID is only required when using the `-o create_bucket` option.
+
 Note: You may also want to create the global credential file first
 
 ```
-echo MYIDENTITY:MYCREDENTIAL > /etc/passwd-s3fs
+echo ACCESS_KEY_ID:SECRET_ACCESS_KEY > /etc/passwd-s3fs
 chmod 600 /etc/passwd-s3fs
 ```
 
@@ -101,24 +170,26 @@ Generally S3 cannot offer the same performance or semantics as a local file syst
 
 * random writes or appends to files require rewriting the entire file
 * metadata operations such as listing directories have poor performance due to network latency
-* [eventual consistency](https://en.wikipedia.org/wiki/Eventual_consistency) can temporarily yield stale data([Amazon S3 Data Consistency Model](http://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel))
+* [eventual consistency](https://en.wikipedia.org/wiki/Eventual_consistency) can temporarily yield stale data([Amazon S3 Data Consistency Model](https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel))
 * no atomic renames of files or directories
 * no coordination between multiple clients mounting the same bucket
 * no hard links
+* inotify detects only local modifications, not external ones by other clients or tools
 
 References
 ----------
 
 * [goofys](https://github.com/kahing/goofys) - similar to s3fs but has better performance and less POSIX compatibility
 * [s3backer](https://github.com/archiecobbs/s3backer) - mount an S3 bucket as a single file
-* [s3fs-python](https://fedorahosted.org/s3fs/) - an older and less complete implementation written in Python
-* [S3Proxy](https://github.com/andrewgaul/s3proxy) - combine with s3fs to mount EMC Atmos, Microsoft Azure, and OpenStack Swift buckets
-* [s3ql](https://bitbucket.org/nikratio/s3ql/) - similar to s3fs but uses its own object format
+* [S3Proxy](https://github.com/gaul/s3proxy) - combine with s3fs to mount Backblaze B2, EMC Atmos, Microsoft Azure, and OpenStack Swift buckets
+* [s3ql](https://github.com/s3ql/s3ql/) - similar to s3fs but uses its own object format
 * [YAS3FS](https://github.com/danilop/yas3fs) - similar to s3fs but uses SNS to allow multiple clients to mount a bucket
 
 Frequently Asked Questions
 --------------------------
 * [FAQ wiki page](https://github.com/s3fs-fuse/s3fs-fuse/wiki/FAQ)
+* [s3fs on Stack Overflow](https://stackoverflow.com/questions/tagged/s3fs)
+* [s3fs on Server Fault](https://serverfault.com/questions/tagged/s3fs)
 
 License
 -------
@@ -126,3 +197,4 @@ License
 Copyright (C) 2010 Randy Rizun <rrizun@gmail.com>
 
 Licensed under the GNU GPL version 2
+

configure.ac

@@ -20,7 +20,7 @@
 dnl Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.59)
-AC_INIT(s3fs, 1.82)
+AC_INIT(s3fs, 1.85)
 AC_CONFIG_HEADER([config.h])
 
 AC_CANONICAL_SYSTEM
@@ -36,9 +36,14 @@ AC_CHECK_HEADERS([sys/extattr.h])
 CXXFLAGS="$CXXFLAGS -Wall -D_FILE_OFFSET_BITS=64"
 
 dnl ----------------------------------------------
-dnl For OSX
+dnl For macOS
 dnl ----------------------------------------------
 case "$target" in
+   *-cygwin* )
+      # Do something specific for windows using winfsp
+      CXXFLAGS="$CXXFLAGS -D_GNU_SOURCE=1"
+      min_fuse_version=2.8
+      ;;
    *-darwin* )
       # Do something specific for mac
       min_fuse_version=2.7.3
@@ -176,13 +181,13 @@ dnl
 dnl For PKG_CONFIG before checking nss/gnutls.
 dnl this is redundant checking, but we need checking before following.
 dnl
-PKG_CHECK_MODULES([common_lib_checking], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6])
+PKG_CHECK_MODULES([common_lib_checking], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 ])
 
 AC_MSG_CHECKING([compile s3fs with])
 case "${auth_lib}" in
 openssl)
   AC_MSG_RESULT(OpenSSL)
-  PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9])
+  PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9 ])
   ;;
 gnutls)
   AC_MSG_RESULT(GnuTLS-gcrypt)
@@ -232,7 +237,7 @@ dnl ----------------------------------------------
 dnl malloc_trim function
 AC_CHECK_FUNCS([malloc_trim])
 
-dnl clock_gettime function(osx)
+dnl clock_gettime function(macos)
 AC_SEARCH_LIBS([clock_gettime],[rt posix4]) 
 AC_CHECK_FUNCS([clock_gettime])
 

doc/man/s3fs.1

@@ -16,10 +16,14 @@ For root.
 For unprivileged user.
 .SS utility mode ( remove interrupted multipart uploading objects )
 .TP
-\fBs3fs \-u bucket
+\fBs3fs --incomplete-mpu-list(-u) bucket
+.TP
+\fBs3fs --incomplete-mpu-abort[=all | =<expire date format>] bucket
 .SH DESCRIPTION
 s3fs is a FUSE filesystem that allows you to mount an Amazon S3 bucket as a local filesystem. It stores files natively and transparently in S3 (i.e., you can use other programs to access the same files).
 .SH AUTHENTICATION
+s3fs supports the standard AWS credentials (filehttps://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) stored in `${HOME}/.aws/credentials`.
+Alternatively, s3fs supports a custom passwd file.
 The s3fs password file has this format (use this format if you have only one set of credentials):
 .RS 4
 \fBaccessKeyId\fP:\fBsecretAccessKey\fP
@@ -60,9 +64,9 @@ if it is not specified bucket name(and path) in command line, must specify this
 \fB\-o\fR default_acl (default="private")
 the default canned acl to apply to all written s3 objects, e.g., "private", "public-read".
 empty string means do not send header.
-see http://aws.amazon.com/documentation/s3/ for the full list of canned acls.
+see https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl for the full list of canned acls.
 .TP
-\fB\-o\fR retries (default="2")
+\fB\-o\fR retries (default="5")
 number of times to retry a failed S3 transaction.
 .TP
 \fB\-o\fR use_cache (default="" which means disabled)
@@ -78,7 +82,7 @@ delete local file cache when s3fs starts and exits.
 \fB\-o\fR storage_class (default is standard)
 store object with specified storage class.
 this option replaces the old option use_rrs.
-Possible values: standard, standard_ia, and reduced_redundancy.
+Possible values: standard, standard_ia, onezone_ia and reduced_redundancy.
 .TP
 \fB\-o\fR use_rrs (default is disable)
 use Amazon's Reduced Redundancy Storage.
@@ -133,6 +137,10 @@ This option specifies the configuration file path which file is the additional H
  A sample configuration file is uploaded in "test" directory.
 If you specify this option for set "Content-Encoding" HTTP header, please take care for RFC 2616.
 .TP
+\fB\-o\fR profile (default="default")
+Choose a profile from ${HOME}/.aws/credentials to authenticate against S3.
+Note that this format matches the AWS CLI format and differs from the s3fs passwd format.
+.TP
 \fB\-o\fR public_bucket (default="" which means disabled)
 anonymously mount a public bucket when set to 1, ignores the $HOME/.passwd-s3fs and /etc/passwd-s3fs files.
 S3 does not allow copy object api for anonymous users, then s3fs sets nocopyapi option automatically when public_bucket=1 option is specified.
@@ -143,7 +151,10 @@ time to wait for connection before giving up.
 \fB\-o\fR readwrite_timeout (default="60" seconds)
 time to wait between read/write activity before giving up.
 .TP
-\fB\-o\fR max_stat_cache_size (default="1000" entries (about 4MB))
+\fB\-o\fR list_object_max_keys (default="1000")
+specify the maximum number of keys returned by S3 list object API. The default is 1000. you can set this value to 1000 or more.
+.TP
+\fB\-o\fR max_stat_cache_size (default="100,000" entries (about 40MB))
 maximum number of entries in the stat cache
 .TP
 \fB\-o\fR stat_cache_expire (default is no expire)
@@ -183,7 +194,7 @@ number of one part size in multipart uploading request.
 The default size is 10MB(10485760byte), minimum value is 5MB(5242880byte).
 Specify number of MB and over 5(MB).
 .TP
-\fB\-o\fR ensure_diskfree(default the same as multipart_size value)
+\fB\-o\fR ensure_diskfree(default 0)
 sets MB to ensure disk free space. This option means the threshold of free space size on disk which is used for the cache file by s3fs.
 s3fs makes file for downloading, and uploading and caching files.
 If the disk free space is smaller than this value, s3fs do not use diskspace as possible in exchange for the performance.
@@ -200,7 +211,7 @@ But if you do not specify this option, and if you can not connect with the defau
 So s3fs can know the correct region name, because s3fs can find it in an error from the S3 server.
 .TP
 \fB\-o\fR sigv2 (default is signature version 4)
-sets signing AWS requests by sing Signature Version 2.
+sets signing AWS requests by using Signature Version 2.
 .TP
 \fB\-o\fR mp_umask (default is "0000")
 sets umask for the mount point directory.
@@ -211,14 +222,21 @@ But if you set the allow_other with this option, you can control permissions of
 \fB\-o\fR nomultipart - disable multipart uploads
 .TP
 \fB\-o\fR enable_content_md5 ( default is disable )
-verifying uploaded data without multipart by content-md5 header.
-Enable to send "Content-MD5" header when uploading a object without multipart posting.
-If this option is enabled, it has some influences on a performance of s3fs when uploading small object.
-Because s3fs always checks MD5 when uploading large object, this option does not affect on large object.
+Allow S3 server to check data integrity of uploads via the Content-MD5 header.
+This can add CPU overhead to transfers.
+.TP
+\fB\-o\fR ecs ( default is disable )
+This option instructs s3fs to query the ECS container credential metadata address instead of the instance metadata address.
 .TP
 \fB\-o\fR iam_role ( default is no IAM role )
 This option requires the IAM role name or "auto". If you specify "auto", s3fs will automatically use the IAM role names that are set to an instance. If you specify this option without any argument, it is the same as that you have specified the "auto".
 .TP
+\fB\-o\fR ibm_iam_auth ( default is not using IBM IAM authentication )
+This option instructs s3fs to use IBM IAM authentication. In this mode, the AWSAccessKey and AWSSecretKey will be used as IBM's Service-Instance-ID and APIKey, respectively.
+.TP
+\fB\-o\fR ibm_iam_endpoint ( default is https://iam.bluemix.net )
+Set the URL to use for IBM IAM authentication.
+.TP
 \fB\-o\fR use_xattr ( default is not handling the extended attribute )
 Enable to handle the extended attribute(xattrs).
 If you set this option, you can use the extended attribute.
@@ -250,6 +268,10 @@ Customize TLS cipher suite list. Expects a colon separated list of cipher suite
 A list of available cipher suites, depending on your TLS engine, can be found on the CURL library documentation:
 https://curl.haxx.se/docs/ssl-ciphers.html
 .TP
+\fB\-o\fR instance_name
+The instance name of the current s3fs mountpoint.
+This name will be added to logging messages and user agent headers sent by s3fs.
+.TP
 \fB\-o\fR complement_stat (complement lack of file/directory mode)
 s3fs complements lack of information about file/directory mode if a file or a directory object does not have x-amz-meta-mode header.
 As default, s3fs does not complements stat information for a object, then the object will not be able to be allowed to list/modify.
@@ -265,6 +287,14 @@ However, if there is a directory object other than "dir/" in the bucket, specify
 s3fs may not be able to recognize the object correctly if an object created by s3fs exists in the bucket.
 Please use this option when the directory in the bucket is only "dir/" object.
 .TP
+\fB\-o\fR use_wtf8 - support arbitrary file system encoding.
+S3 requires all object names to be valid utf-8. But some
+clients, notably Windows NFS clients, use their own encoding.
+This option re-encodes invalid utf-8 object names into valid
+utf-8 by mapping offending codes into a 'private' codepage of the
+Unicode set.
+Useful on clients not using utf-8 as their file system encoding.
+.TP
 \fB\-o\fR dbglevel (default="crit")
 Set the debug message level. set value as crit(critical), err(error), warn(warning), info(information) to debug level. default debug level is critical.
 If s3fs run with "-d" option, the debug level is set information.
@@ -272,6 +302,18 @@ When s3fs catch the signal SIGUSR2, the debug level is bumpup.
 .TP
 \fB\-o\fR curldbg - put curl debug message
 Put the debug message from libcurl when this option is specified.
+.SS "utility mode options"
+.TP
+\fB\-u\fR or \fB\-\-incomplete\-mpu\-list\fR
+Lists multipart incomplete objects uploaded to the specified bucket.
+.TP
+\fB\-\-incomplete\-mpu\-abort\fR all or date format(default="24H")
+Delete the multipart incomplete object uploaded to the specified bucket.
+If "all" is specified for this option, all multipart incomplete objects will be deleted.
+If you specify no argument as an option, objects older than 24 hours(24H) will be deleted(This is the default value).
+You can specify an optional date format.
+It can be specified as year, month, day, hour, minute, second, and it is expressed as "Y", "M", "D", "h", "m", "s" respectively.
+For example, "1Y6M10D12h30m30s".
 .SH FUSE/MOUNT OPTIONS
 .TP
 Most of the generic mount options described in 'man mount' are supported (ro, rw, suid, nosuid, dev, nodev, exec, noexec, atime, noatime, sync async, dirsync).  Filesystems are mounted with '\-onodev,nosuid' by default, which can only be overridden by a privileged user.
@@ -279,7 +321,7 @@ Most of the generic mount options described in 'man mount' are supported (ro, rw
 There are many FUSE specific mount options that can be specified. e.g. allow_other. See the FUSE README for the full set.
 .SH NOTES
 .TP
-The maximum size of objects that s3fs can handle depends on Amazone S3. For example, up to 5 GB when using single PUT API. And up to 5 TB is supported when Multipart Upload API is used.
+The maximum size of objects that s3fs can handle depends on Amazon S3. For example, up to 5 GB when using single PUT API. And up to 5 TB is supported when Multipart Upload API is used.
 .TP
 If enabled via the "use_cache" option, s3fs automatically maintains a local cache of files in the folder specified by use_cache. Whenever s3fs needs to read or write a file on S3, it first downloads the entire file locally to the folder specified by use_cache and operates on it. When fuse_release() is called, s3fs will re-upload the file to S3 if it has been changed. s3fs uses md5 checksums to minimize downloads from S3.
 .TP

src/addhead.cpp

@@ -22,7 +22,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <syslog.h>
-#include <assert.h>
 #include <curl/curl.h>
 #include <sstream>
 #include <fstream>
@@ -56,7 +55,7 @@ AdditionalHeader::AdditionalHeader()
   if(this == AdditionalHeader::get()){
     is_enable = false;
   }else{
-    assert(false);
+    abort();
   }
 }
 
@@ -65,7 +64,7 @@ AdditionalHeader::~AdditionalHeader()
   if(this == AdditionalHeader::get()){
     Unload();
   }else{
-    assert(false);
+    abort();
   }
 }
 
@@ -90,14 +89,14 @@ bool AdditionalHeader::Load(const char* file)
     if('#' == line[0]){
       continue;
     }
-    if(0 == line.size()){
+    if(line.empty()){
       continue;
     }
     // load a line
-    stringstream ss(line);
-    string       key("");       // suffix(key)
-    string       head;          // additional HTTP header
-    string       value;         // header value
+    istringstream ss(line);
+    string        key;           // suffix(key)
+    string        head;          // additional HTTP header
+    string        value;         // header value
     if(0 == isblank(line[0])){
       ss >> key;
     }
@@ -109,8 +108,8 @@ bool AdditionalHeader::Load(const char* file)
     }
 
     // check it
-    if(0 == head.size()){
-      if(0 == key.size()){
+    if(head.empty()){
+      if(key.empty()){
         continue;
       }
       S3FS_PRN_ERR("file format error: %s key(suffix) is no HTTP header value.", key.c_str());
@@ -123,6 +122,7 @@ bool AdditionalHeader::Load(const char* file)
       // regex
       if(key.size() <= strlen(ADD_HEAD_REGEX)){
         S3FS_PRN_ERR("file format error: %s key(suffix) does not have key string.", key.c_str());
+        delete paddhead;
         continue;
       }
       key = key.substr(strlen(ADD_HEAD_REGEX));
@@ -130,8 +130,8 @@ bool AdditionalHeader::Load(const char* file)
       // compile
       regex_t*  preg = new regex_t;
       int       result;
-      char      errbuf[256];
       if(0 != (result = regcomp(preg, key.c_str(), REG_EXTENDED | REG_NOSUB))){ // we do not need matching info
+        char    errbuf[256];
         regerror(result, preg, errbuf, sizeof(errbuf));
         S3FS_PRN_ERR("failed to compile regex from %s key by %s.", key.c_str(), errbuf);
         delete preg;
@@ -164,7 +164,7 @@ bool AdditionalHeader::Load(const char* file)
   return true;
 }
 
-void AdditionalHeader::Unload(void)
+void AdditionalHeader::Unload()
 {
   is_enable = false;
 
@@ -239,14 +239,14 @@ struct curl_slist* AdditionalHeader::AddHeader(struct curl_slist* list, const ch
   return list;
 }
 
-bool AdditionalHeader::Dump(void) const
+bool AdditionalHeader::Dump() const
 {
   if(!IS_S3FS_LOG_DBG()){
     return true;
   }
 
-  stringstream ssdbg;
-  int          cnt = 1;
+  ostringstream ssdbg;
+  int           cnt = 1;
 
   ssdbg << "Additional Header list[" << addheadlist.size() << "] = {" << endl;
 

src/cache.cpp

@@ -28,7 +28,6 @@
 #include <stdint.h>
 #include <pthread.h>
 #include <string.h>
-#include <assert.h>
 #include <syslog.h>
 #include <string>
 #include <map>
@@ -59,7 +58,7 @@ using namespace std;
 #ifdef HAVE_CLOCK_GETTIME
 static int s3fs_clock_gettime(int clk_id, struct timespec* ts)
 {
-  return clock_gettime(clk_id, ts);
+  return clock_gettime(static_cast<clockid_t>(clk_id), ts);
 }
 #else
 static int s3fs_clock_gettime(int clk_id, struct timespec* ts)
@@ -142,13 +141,16 @@ pthread_mutex_t StatCache::stat_cache_lock;
 //-------------------------------------------------------------------
 // Constructor/Destructor
 //-------------------------------------------------------------------
-StatCache::StatCache() : IsExpireTime(false), IsExpireIntervalType(false), ExpireTime(0), CacheSize(1000), IsCacheNoObject(false)
+StatCache::StatCache() : IsExpireTime(false), IsExpireIntervalType(false), ExpireTime(0), CacheSize(100000), IsCacheNoObject(false)
 {
   if(this == StatCache::getStatCacheData()){
     stat_cache.clear();
-    pthread_mutex_init(&(StatCache::stat_cache_lock), NULL);
+    pthread_mutexattr_t attr;
+    pthread_mutexattr_init(&attr);
+    pthread_mutexattr_settype(&attr, S3FS_MUTEX_RECURSIVE);
+    pthread_mutex_init(&StatCache::stat_cache_lock, &attr);
   }else{
-    assert(false);
+    abort();
   }
 }
 
@@ -156,16 +158,16 @@ StatCache::~StatCache()
 {
   if(this == StatCache::getStatCacheData()){
     Clear();
-    pthread_mutex_destroy(&(StatCache::stat_cache_lock));
+    pthread_mutex_destroy(&StatCache::stat_cache_lock);
   }else{
-    assert(false);
+    abort();
   }
 }
 
 //-------------------------------------------------------------------
 // Methods
 //-------------------------------------------------------------------
-unsigned long StatCache::GetCacheSize(void) const
+unsigned long StatCache::GetCacheSize() const
 {
   return CacheSize;
 }
@@ -177,7 +179,7 @@ unsigned long StatCache::SetCacheSize(unsigned long size)
   return old;
 }
 
-time_t StatCache::GetExpireTime(void) const
+time_t StatCache::GetExpireTime() const
 {
   return (IsExpireTime ? ExpireTime : (-1));
 }
@@ -191,7 +193,7 @@ time_t StatCache::SetExpireTime(time_t expire, bool is_interval)
   return old;
 }
 
-time_t StatCache::UnsetExpireTime(void)
+time_t StatCache::UnsetExpireTime()
 {
   time_t old           = IsExpireTime ? ExpireTime : (-1);
   ExpireTime           = 0;
@@ -207,18 +209,15 @@ bool StatCache::SetCacheNoObject(bool flag)
   return old;
 }
 
-void StatCache::Clear(void)
+void StatCache::Clear()
 {
-  pthread_mutex_lock(&StatCache::stat_cache_lock);
+  AutoLock lock(&StatCache::stat_cache_lock);
 
-  for(stat_cache_t::iterator iter = stat_cache.begin(); iter != stat_cache.end(); stat_cache.erase(iter++)){
-    if((*iter).second){
-      delete (*iter).second;
-    }
+  for(stat_cache_t::iterator iter = stat_cache.begin(); iter != stat_cache.end(); ++iter){
+    delete (*iter).second;
   }
+  stat_cache.clear();
   S3FS_MALLOCTRIM(0);
-
-  pthread_mutex_unlock(&StatCache::stat_cache_lock);
 }
 
 bool StatCache::GetStat(string& key, struct stat* pst, headers_t* meta, bool overcheck, const char* petag, bool* pisforce)
@@ -226,23 +225,22 @@ bool StatCache::GetStat(string& key, struct stat* pst, headers_t* meta, bool ove
   bool is_delete_cache = false;
   string strpath = key;
 
-  pthread_mutex_lock(&StatCache::stat_cache_lock);
+  AutoLock lock(&StatCache::stat_cache_lock);
 
   stat_cache_t::iterator iter = stat_cache.end();
   if(overcheck && '/' != strpath[strpath.length() - 1]){
     strpath += "/";
-    iter = stat_cache.find(strpath.c_str());
+    iter = stat_cache.find(strpath);
   }
   if(iter == stat_cache.end()){
     strpath = key;
-    iter = stat_cache.find(strpath.c_str());
+    iter = stat_cache.find(strpath);
   }
 
   if(iter != stat_cache.end() && (*iter).second){
     stat_cache_entry* ent = (*iter).second;
     if(!IsExpireTime || !IsExpireStatCacheTime(ent->cache_date, ExpireTime)){
       if(ent->noobjcache){
-        pthread_mutex_unlock(&StatCache::stat_cache_lock);
         if(!IsCacheNoObject){
           // need to delete this cache.
           DelStat(strpath);
@@ -255,10 +253,10 @@ bool StatCache::GetStat(string& key, struct stat* pst, headers_t* meta, bool ove
       string stretag;
       if(petag){
         // find & check ETag
-        for(headers_t::iterator iter = ent->meta.begin(); iter != ent->meta.end(); ++iter){
-          string tag = lower(iter->first);
+        for(headers_t::iterator hiter = ent->meta.begin(); hiter != ent->meta.end(); ++hiter){
+          string tag = lower(hiter->first);
           if(tag == "etag"){
-            stretag = iter->second;
+            stretag = hiter->second;
             if('\0' != petag[0] && 0 != strcmp(petag, stretag.c_str())){
               is_delete_cache = true;
             }
@@ -289,7 +287,6 @@ bool StatCache::GetStat(string& key, struct stat* pst, headers_t* meta, bool ove
         if(IsExpireIntervalType){
           SetStatCacheTime(ent->cache_date);
         }
-        pthread_mutex_unlock(&StatCache::stat_cache_lock);
         return true;
       }
 
@@ -298,7 +295,6 @@ bool StatCache::GetStat(string& key, struct stat* pst, headers_t* meta, bool ove
       is_delete_cache = true;
     }
   }
-  pthread_mutex_unlock(&StatCache::stat_cache_lock);
 
   if(is_delete_cache){
     DelStat(strpath);
@@ -315,16 +311,16 @@ bool StatCache::IsNoObjectCache(string& key, bool overcheck)
     return false;
   }
 
-  pthread_mutex_lock(&StatCache::stat_cache_lock);
+  AutoLock lock(&StatCache::stat_cache_lock);
 
   stat_cache_t::iterator iter = stat_cache.end();
   if(overcheck && '/' != strpath[strpath.length() - 1]){
     strpath += "/";
-    iter = stat_cache.find(strpath.c_str());
+    iter = stat_cache.find(strpath);
   }
   if(iter == stat_cache.end()){
     strpath = key;
-    iter = stat_cache.find(strpath.c_str());
+    iter = stat_cache.find(strpath);
   }
 
   if(iter != stat_cache.end() && (*iter).second) {
@@ -332,7 +328,6 @@ bool StatCache::IsNoObjectCache(string& key, bool overcheck)
       if((*iter).second->noobjcache){
         // noobjcache = true means no object.
         SetStatCacheTime((*iter).second->cache_date);
-        pthread_mutex_unlock(&StatCache::stat_cache_lock);
         return true;
       }
     }else{
@@ -340,7 +335,6 @@ bool StatCache::IsNoObjectCache(string& key, bool overcheck)
       is_delete_cache = true;
     }
   }
-  pthread_mutex_unlock(&StatCache::stat_cache_lock);
 
   if(is_delete_cache){
     DelStat(strpath);
@@ -355,12 +349,13 @@ bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir, bool n
   }
   S3FS_PRN_INFO3("add stat cache entry[path=%s]", key.c_str());
 
-  pthread_mutex_lock(&StatCache::stat_cache_lock);
-
-  bool found = stat_cache.end() != stat_cache.find(key);
-  bool do_truncate = stat_cache.size() > CacheSize;
-
-  pthread_mutex_unlock(&StatCache::stat_cache_lock);
+  bool found;
+  bool do_truncate;
+  {
+    AutoLock lock(&StatCache::stat_cache_lock);
+    found = stat_cache.end() != stat_cache.find(key);
+    do_truncate = stat_cache.size() > CacheSize;
+  }
 
   if(found){
     DelStat(key.c_str());
@@ -402,19 +397,15 @@ bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir, bool n
   }
 
   // add
-  pthread_mutex_lock(&StatCache::stat_cache_lock);
+  AutoLock lock(&StatCache::stat_cache_lock);
 
   stat_cache_t::iterator iter = stat_cache.find(key);   // recheck for same key exists
   if(stat_cache.end() != iter){
-    if(iter->second){
-      delete iter->second;
-    }
+    delete iter->second;
     stat_cache.erase(iter);
   }
   stat_cache[key] = ent;
 
-  pthread_mutex_unlock(&StatCache::stat_cache_lock);
-
   return true;
 }
 
@@ -428,12 +419,13 @@ bool StatCache::AddNoObjectCache(string& key)
   }
   S3FS_PRN_INFO3("add no object cache entry[path=%s]", key.c_str());
 
-  pthread_mutex_lock(&StatCache::stat_cache_lock);
-
-  bool found = stat_cache.end() != stat_cache.find(key);
-  bool do_truncate = stat_cache.size() > CacheSize;
-
-  pthread_mutex_unlock(&StatCache::stat_cache_lock);
+  bool found;
+  bool do_truncate;
+  {
+    AutoLock lock(&StatCache::stat_cache_lock);
+    found = stat_cache.end() != stat_cache.find(key);
+    do_truncate = stat_cache.size() > CacheSize;
+  }
 
   if(found){
     DelStat(key.c_str());
@@ -456,26 +448,21 @@ bool StatCache::AddNoObjectCache(string& key)
   SetStatCacheTime(ent->cache_date);    // Set time.
 
   // add
-  pthread_mutex_lock(&StatCache::stat_cache_lock);
+  AutoLock lock(&StatCache::stat_cache_lock);
 
   stat_cache_t::iterator iter = stat_cache.find(key);   // recheck for same key exists
   if(stat_cache.end() != iter){
-    if(iter->second){
-      delete iter->second;
-    }
+    delete iter->second;
     stat_cache.erase(iter);
   }
   stat_cache[key] = ent;
 
-  pthread_mutex_unlock(&StatCache::stat_cache_lock);
-
   return true;
 }
 
-void StatCache::ChangeNoTruncateFlag(std::string key, bool no_truncate)
+void StatCache::ChangeNoTruncateFlag(const std::string& key, bool no_truncate)
 {
-  pthread_mutex_lock(&StatCache::stat_cache_lock);
-
+  AutoLock lock(&StatCache::stat_cache_lock);
   stat_cache_t::iterator iter = stat_cache.find(key);
 
   if(stat_cache.end() != iter){
@@ -490,25 +477,22 @@ void StatCache::ChangeNoTruncateFlag(std::string key, bool no_truncate)
       }
     }
   }
-  pthread_mutex_unlock(&StatCache::stat_cache_lock);
 }
 
-bool StatCache::TruncateCache(void)
+bool StatCache::TruncateCache()
 {
+  AutoLock lock(&StatCache::stat_cache_lock);
+
   if(stat_cache.empty()){
     return true;
   }
 
-  pthread_mutex_lock(&StatCache::stat_cache_lock);
-
   // 1) erase over expire time
   if(IsExpireTime){
     for(stat_cache_t::iterator iter = stat_cache.begin(); iter != stat_cache.end(); ){
       stat_cache_entry* entry = iter->second;
       if(!entry || (0L == entry->notruncate && IsExpireStatCacheTime(entry->cache_date, ExpireTime))){
-        if(entry){
-            delete entry;
-        }
+        delete entry;
         stat_cache.erase(iter++);
       }else{
         ++iter;
@@ -518,7 +502,6 @@ bool StatCache::TruncateCache(void)
 
   // 2) check stat cache count
   if(stat_cache.size() < CacheSize){
-    pthread_mutex_unlock(&StatCache::stat_cache_lock);
     return true;
   }
 
@@ -545,15 +528,11 @@ bool StatCache::TruncateCache(void)
     stat_cache_t::iterator siter = *iiter;
 
     S3FS_PRN_DBG("truncate stat cache[path=%s]", siter->first.c_str());
-    if(siter->second){
-        delete siter->second;
-    }
+    delete siter->second;
     stat_cache.erase(siter);
   }
   S3FS_MALLOCTRIM(0);
 
-  pthread_mutex_unlock(&StatCache::stat_cache_lock);
-
   return true;
 }
 
@@ -564,13 +543,11 @@ bool StatCache::DelStat(const char* key)
   }
   S3FS_PRN_INFO3("delete stat cache entry[path=%s]", key);
 
-  pthread_mutex_lock(&StatCache::stat_cache_lock);
+  AutoLock lock(&StatCache::stat_cache_lock);
 
   stat_cache_t::iterator iter;
   if(stat_cache.end() != (iter = stat_cache.find(string(key)))){
-    if((*iter).second){
-      delete (*iter).second;
-    }
+    delete (*iter).second;
     stat_cache.erase(iter);
   }
   if(0 < strlen(key) && 0 != strcmp(key, "/")){
@@ -582,17 +559,13 @@ bool StatCache::DelStat(const char* key)
       // If there is "path/" cache, delete it.
       strpath += "/";
     }
-    if(stat_cache.end() != (iter = stat_cache.find(strpath.c_str()))){
-      if((*iter).second){
-        delete (*iter).second;
-      }
+    if(stat_cache.end() != (iter = stat_cache.find(strpath))){
+      delete (*iter).second;
       stat_cache.erase(iter);
     }
   }
   S3FS_MALLOCTRIM(0);
 
-  pthread_mutex_unlock(&StatCache::stat_cache_lock);
-
   return true;
 }
 
@@ -620,6 +593,9 @@ bool convert_header_to_stat(const char* path, headers_t& meta, struct stat* pst,
   // mtime
   pst->st_mtime = get_mtime(meta);
 
+  // ctime
+  pst->st_ctime = get_ctime(meta);
+
   // size
   pst->st_size = get_size(meta);
 

src/cache.h

@@ -117,7 +117,7 @@ class StatCache
     bool AddStat(std::string& key, headers_t& meta, bool forcedir = false, bool no_truncate = false);
 
     // Change no truncate flag
-    void ChangeNoTruncateFlag(std::string key, bool no_truncate);
+    void ChangeNoTruncateFlag(const std::string& key, bool no_truncate);
 
     // Delete stat cache
     bool DelStat(const char* key);

src/common.h

@@ -21,6 +21,7 @@
 #ifndef S3FS_COMMON_H_
 #define S3FS_COMMON_H_
 
+#include <stdlib.h>
 #include "../config.h"
 
 //
@@ -37,7 +38,7 @@
 //
 // Macro
 //
-#define SAFESTRPTR(strptr) (strptr ? strptr : "")
+static inline const char *SAFESTRPTR(const char *strptr) { return strptr ? strptr : ""; }
 
 //
 // Debug level
@@ -79,7 +80,7 @@ enum s3fs_log_level{
          if(foreground){ \
            fprintf(stdout, "%s%s:%s(%d): " fmt "%s\n", S3FS_LOG_LEVEL_STRING(level), __FILE__, __func__, __LINE__, __VA_ARGS__); \
          }else{ \
-           syslog(S3FS_LOG_LEVEL_TO_SYSLOG(level), "%s:%s(%d): " fmt "%s", __FILE__, __func__, __LINE__, __VA_ARGS__); \
+           syslog(S3FS_LOG_LEVEL_TO_SYSLOG(level), "%s%s:%s(%d): " fmt "%s", instance_name.c_str(), __FILE__, __func__, __LINE__, __VA_ARGS__); \
          } \
        }
 
@@ -88,7 +89,7 @@ enum s3fs_log_level{
          if(foreground){ \
            fprintf(stdout, "%s%s%s:%s(%d): " fmt "%s\n", S3FS_LOG_LEVEL_STRING(level), S3FS_LOG_NEST(nest), __FILE__, __func__, __LINE__, __VA_ARGS__); \
          }else{ \
-           syslog(S3FS_LOG_LEVEL_TO_SYSLOG(level), "%s" fmt "%s", S3FS_LOG_NEST(nest), __VA_ARGS__); \
+           syslog(S3FS_LOG_LEVEL_TO_SYSLOG(level), "%s%s" fmt "%s", instance_name.c_str(), S3FS_LOG_NEST(nest), __VA_ARGS__); \
          } \
        }
 
@@ -97,7 +98,15 @@ enum s3fs_log_level{
          fprintf(stderr, "s3fs: " fmt "%s\n", __VA_ARGS__); \
        }else{ \
          fprintf(stderr, "s3fs: " fmt "%s\n", __VA_ARGS__); \
-         syslog(S3FS_LOG_LEVEL_TO_SYSLOG(S3FS_LOG_CRIT), "s3fs: " fmt "%s", __VA_ARGS__); \
+         syslog(S3FS_LOG_LEVEL_TO_SYSLOG(S3FS_LOG_CRIT), "%ss3fs: " fmt "%s", instance_name.c_str(), __VA_ARGS__); \
+       }
+
+// Special macro for init message
+#define S3FS_PRN_INIT_INFO(fmt, ...) \
+       if(foreground){ \
+         fprintf(stdout, "%s%s%s:%s(%d): " fmt "%s\n", S3FS_LOG_LEVEL_STRING(S3FS_LOG_INFO), S3FS_LOG_NEST(0), __FILE__, __func__, __LINE__, __VA_ARGS__, ""); \
+       }else{ \
+         syslog(S3FS_LOG_LEVEL_TO_SYSLOG(S3FS_LOG_INFO), "%s%s" fmt "%s", instance_name.c_str(), S3FS_LOG_NEST(0), __VA_ARGS__, ""); \
        }
 
 // [NOTE]
@@ -149,6 +158,7 @@ typedef std::map<std::string, PXATTRVAL> xattrs_t;
 //
 // Global variables
 //
+// TODO: namespace these
 extern bool           foreground;
 extern bool           nomultipart;
 extern bool           pathrequeststyle;
@@ -160,6 +170,7 @@ extern std::string    bucket;
 extern std::string    mount_prefix;
 extern std::string    endpoint;
 extern std::string    cipher_suites;
+extern std::string    instance_name;
 extern s3fs_log_level debug_level;
 extern const char*    s3fs_log_nest[S3FS_LOG_NEST_MAX];
 

src/common_auth.cpp

@@ -71,7 +71,6 @@ string s3fs_sha256sum(int fd, off_t start, ssize_t size)
 {
   size_t digestlen = get_sha256_digest_length();
   char sha256[2 * digestlen + 1];
-  char hexbuf[3];
   unsigned char* sha256hex;
 
   if(NULL == (sha256hex = s3fs_sha256hexsum(fd, start, size))){
@@ -80,8 +79,7 @@ string s3fs_sha256sum(int fd, off_t start, ssize_t size)
 
   memset(sha256, 0, 2 * digestlen + 1);
   for(size_t pos = 0; pos < digestlen; pos++){
-    snprintf(hexbuf, 3, "%02x", sha256hex[pos]);
-    strncat(sha256, hexbuf, 2);
+    snprintf(sha256 + 2 * pos, 3, "%02x", sha256hex[pos]);
   }
   free(sha256hex);
 

src/curl.h

@@ -23,10 +23,12 @@
 
 #include <cassert>
 
+#include "psemaphore.h"
+
 //----------------------------------------------
 // Symbols
 //----------------------------------------------
-#define MIN_MULTIPART_SIZE          5242880           // 5MB
+static const int MIN_MULTIPART_SIZE = 5 * 1024 * 1024;
 
 //----------------------------------------------
 // class BodyData
@@ -126,14 +128,12 @@ class S3fsMultiCurl;
 //----------------------------------------------
 // class CurlHandlerPool
 //----------------------------------------------
+typedef std::list<CURL*>            hcurllist_t;
 
 class CurlHandlerPool
 {
 public:
-  explicit CurlHandlerPool(int maxHandlers)
-    : mMaxHandlers(maxHandlers)
-    , mHandlers(NULL)
-    , mIndex(-1)
+  explicit CurlHandlerPool(int maxHandlers) : mMaxHandlers(maxHandlers)
   {
     assert(maxHandlers > 0);
   }
@@ -141,20 +141,23 @@ public:
   bool Init();
   bool Destroy();
 
-  CURL* GetHandler();
-  void ReturnHandler(CURL* h);
+  CURL* GetHandler(bool only_pool);
+  void ReturnHandler(CURL* hCurl, bool restore_pool);
 
 private:
-  int mMaxHandlers;
-
+  int             mMaxHandlers;
   pthread_mutex_t mLock;
-  CURL** mHandlers;
-  int mIndex;
+  hcurllist_t     mPool;
 };
 
 //----------------------------------------------
 // class S3fsCurl
 //----------------------------------------------
+class S3fsCurl;
+
+// Prototype function for lazy setup options for curl handle
+typedef bool (*s3fscurl_lazy_setup)(S3fsCurl* s3fscurl);
+
 typedef std::map<std::string, std::string> iamcredmap_t;
 typedef std::map<std::string, std::string> sseckeymap_t;
 typedef std::list<sseckeymap_t>            sseckeylist_t;
@@ -163,6 +166,7 @@ typedef std::list<sseckeymap_t>            sseckeylist_t;
 enum storage_class_t {
   STANDARD,
   STANDARD_IA,
+  ONEZONE_IA,
   REDUCED_REDUNDANCY
 };
 
@@ -175,9 +179,11 @@ enum sse_type_t {
 };
 
 // share
-#define	SHARE_MUTEX_DNS         0
-#define	SHARE_MUTEX_SSL_SESSION 1
-#define	SHARE_MUTEX_MAX         2
+enum {
+  SHARE_MUTEX_DNS = 0,
+  SHARE_MUTEX_SSL_SESSION = 1,
+  SHARE_MUTEX_MAX = 2,
+};
 
 // Class for lapping curl
 //
@@ -230,13 +236,21 @@ class S3fsCurl
     static std::string      AWSSecretAccessKey;
     static std::string      AWSAccessToken;
     static time_t           AWSAccessTokenExpire;
+    static bool             is_ecs;
+    static bool             is_ibm_iam_auth;
+    static std::string      IAM_cred_url;
+    static size_t           IAM_field_count;
+    static std::string      IAM_token_field;
+    static std::string      IAM_expiry_field;
     static std::string      IAM_role;
     static long             ssl_verify_hostname;
     static curltime_t       curl_times;
     static curlprogress_t   curl_progress;
     static std::string      curl_ca_bundle;
     static mimes_t          mimeTypes;
+    static std::string      userAgent;
     static int              max_parallel_cnt;
+    static int              max_multireq;
     static off_t            multipart_size;
     static bool             is_sigv4;
     static bool             is_ua;             // User-Agent
@@ -266,6 +280,12 @@ class S3fsCurl
     int                  b_ssekey_pos;         // backup for retrying
     std::string          b_ssevalue;           // backup for retrying
     sse_type_t           b_ssetype;            // backup for retrying
+    std::string          op;                   // the HTTP verb of the request ("PUT", "GET", etc.)
+    std::string          query_string;         // request query string
+    Semaphore            *sem;
+    pthread_mutex_t      *completed_tids_lock;
+    std::vector<pthread_t> *completed_tids;
+    s3fscurl_lazy_setup  fpLazySetup;          // curl options for lazy setting function
 
   public:
     // constructor/destructor
@@ -293,9 +313,17 @@ class S3fsCurl
     static size_t DownloadWriteCallback(void* ptr, size_t size, size_t nmemb, void* userp);
 
     static bool UploadMultipartPostCallback(S3fsCurl* s3fscurl);
+    static bool CopyMultipartPostCallback(S3fsCurl* s3fscurl);
     static S3fsCurl* UploadMultipartPostRetryCallback(S3fsCurl* s3fscurl);
+    static S3fsCurl* CopyMultipartPostRetryCallback(S3fsCurl* s3fscurl);
     static S3fsCurl* ParallelGetObjectRetryCallback(S3fsCurl* s3fscurl);
 
+    // lazy functions for set curl options
+    static bool UploadMultipartPostSetCurlOpts(S3fsCurl* s3fscurl);
+    static bool CopyMultipartPostSetCurlOpts(S3fsCurl* s3fscurl);
+    static bool PreGetObjectRequestSetCurlOpts(S3fsCurl* s3fscurl);
+    static bool PreHeadRequestSetCurlOpts(S3fsCurl* s3fscurl);
+
     static bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval);
     static bool SetIAMCredentials(const char* response);
     static bool ParseIAMRoleFromMetaDataResponse(const char* response, std::string& rolename);
@@ -311,15 +339,19 @@ class S3fsCurl
     bool ResetHandle(void);
     bool RemakeHandle(void);
     bool ClearInternalData(void);
-    void insertV4Headers(const std::string &op, const std::string &path, const std::string &query_string, const std::string &payload_hash);
+    void insertV4Headers();
+    void insertV2Headers();
+    void insertIBMIAMHeaders();
+    void insertAuthHeaders();
     std::string CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource);
     std::string CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601);
     bool GetUploadId(std::string& upload_id);
     int GetIAMCredentials(void);
 
     int UploadMultipartPostSetup(const char* tpath, int part_num, const std::string& upload_id);
-    int CopyMultipartPostRequest(const char* from, const char* to, int part_num, std::string& upload_id, headers_t& meta);
+    int CopyMultipartPostSetup(const char* from, const char* to, int part_num, std::string& upload_id, headers_t& meta);
     bool UploadMultipartPostComplete();
+    bool CopyMultipartPostComplete();
 
   public:
     // class methods
@@ -341,6 +373,7 @@ class S3fsCurl
     static bool SetPublicBucket(bool flag);
     static bool IsPublicBucket(void) { return S3fsCurl::is_public_bucket; }
     static std::string SetDefaultAcl(const char* acl);
+    static std::string GetDefaultAcl();
     static storage_class_t SetStorageClass(storage_class_t storage_class);
     static storage_class_t GetStorageClass() { return S3fsCurl::storage_class; }
     static bool LoadEnvSse(void) { return (S3fsCurl::LoadEnvSseCKeys() && S3fsCurl::LoadEnvSseKmsid()); }
@@ -362,13 +395,26 @@ class S3fsCurl
     static bool SetVerbose(bool flag);
     static bool GetVerbose(void) { return S3fsCurl::is_verbose; }
     static bool SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey);
-    static bool IsSetAccessKeyId(void){
-                  return (0 < S3fsCurl::IAM_role.size() || (0 < S3fsCurl::AWSAccessKeyId.size() && 0 < S3fsCurl::AWSSecretAccessKey.size()));
+    static bool IsSetAccessKeyID(void){
+                  return (0 < S3fsCurl::AWSAccessKeyId.size());
+                }
+    static bool IsSetAccessKeys(void){
+                  return (0 < S3fsCurl::IAM_role.size() || ((0 < S3fsCurl::AWSAccessKeyId.size() || S3fsCurl::is_ibm_iam_auth) && 0 < S3fsCurl::AWSSecretAccessKey.size()));
                 }
     static long SetSslVerifyHostname(long value);
     static long GetSslVerifyHostname(void) { return S3fsCurl::ssl_verify_hostname; }
+    // maximum parallel GET and PUT requests
     static int SetMaxParallelCount(int value);
     static int GetMaxParallelCount(void) { return S3fsCurl::max_parallel_cnt; }
+    // maximum parallel HEAD requests
+    static int SetMaxMultiRequest(int max);
+    static int GetMaxMultiRequest(void) { return S3fsCurl::max_multireq; }
+    static bool SetIsECS(bool flag);
+    static bool SetIsIBMIAMAuth(bool flag);
+    static size_t SetIAMFieldCount(size_t field_count);
+    static std::string SetIAMCredentialsURL(const char* url);
+    static std::string SetIAMTokenField(const char* token_field);
+    static std::string SetIAMExpiryField(const char* expiry_field);
     static std::string SetIAMRole(const char* role);
     static const char* GetIAMRole(void) { return S3fsCurl::IAM_role.c_str(); }
     static bool SetMultipartSize(off_t size);
@@ -377,14 +423,15 @@ class S3fsCurl
     static bool IsSignatureV4(void) { return S3fsCurl::is_sigv4; }
     static bool SetUserAgentFlag(bool isset) { bool bresult = S3fsCurl::is_ua; S3fsCurl::is_ua = isset; return bresult; }
     static bool IsUserAgentFlag(void) { return S3fsCurl::is_ua; }
+    static void InitUserAgent(void);
 
     // methods
-    bool CreateCurlHandle(bool force = false);
-    bool DestroyCurlHandle(void);
+    bool CreateCurlHandle(bool only_pool = false, bool remake = false);
+    bool DestroyCurlHandle(bool restore_pool = true, bool clear_internal_data = true);
 
     bool LoadIAMRoleFromMetaData(void);
     bool AddSseRequestHead(sse_type_t ssetype, std::string& ssevalue, bool is_only_c, bool is_copy);
-    bool GetResponseCode(long& responseCode);
+    bool GetResponseCode(long& responseCode, bool from_curl_handle = true);
     int RequestPerform(void);
     int DeleteRequest(const char* tpath);
     bool PreHeadRequest(const char* tpath, const char* bpath = NULL, const char* savedpath = NULL, int ssekey_pos = -1);
@@ -414,6 +461,7 @@ class S3fsCurl
     std::string GetBasePath(void) const { return base_path; }
     std::string GetSpacialSavedPath(void) const { return saved_path; }
     std::string GetUrl(void) const { return url; }
+    std::string GetOp(void) const { return op; }
     headers_t* GetResponseHeaders(void) { return &responseHeaders; }
     BodyData* GetBodyData(void) const { return bodydata; }
     BodyData* GetHeadData(void) const { return headdata; }
@@ -433,21 +481,24 @@ class S3fsCurl
 //----------------------------------------------
 // Class for lapping multi curl
 //
-typedef std::map<CURL*, S3fsCurl*> s3fscurlmap_t;
+typedef std::vector<S3fsCurl*>       s3fscurllist_t;
 typedef bool (*S3fsMultiSuccessCallback)(S3fsCurl* s3fscurl);    // callback for succeed multi request
 typedef S3fsCurl* (*S3fsMultiRetryCallback)(S3fsCurl* s3fscurl); // callback for failure and retrying
 
 class S3fsMultiCurl
 {
   private:
-    static int    max_multireq;
+    const int maxParallelism;
 
-    s3fscurlmap_t cMap_all;  // all of curl requests
-    s3fscurlmap_t cMap_req;  // curl requests are sent
+    s3fscurllist_t clist_all;  // all of curl requests
+    s3fscurllist_t clist_req;  // curl requests are sent
 
     S3fsMultiSuccessCallback SuccessCallback;
     S3fsMultiRetryCallback   RetryCallback;
 
+    pthread_mutex_t completed_tids_lock;
+    std::vector<pthread_t> completed_tids;
+
   private:
     bool ClearEx(bool is_all);
     int MultiPerform(void);
@@ -456,11 +507,10 @@ class S3fsMultiCurl
     static void* RequestPerformWrapper(void* arg);
 
   public:
-    S3fsMultiCurl();
+    explicit S3fsMultiCurl(int maxParallelism);
     ~S3fsMultiCurl();
 
-    static int SetMaxMultiRequest(int max);
-    static int GetMaxMultiRequest(void) { return S3fsMultiCurl::max_multireq; }
+    int GetMaxParallelism() { return maxParallelism; }
 
     S3fsMultiSuccessCallback SetSuccessCallback(S3fsMultiSuccessCallback function);
     S3fsMultiRetryCallback SetRetryCallback(S3fsMultiRetryCallback function);
@@ -479,6 +529,7 @@ struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* d
 struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* key, const char* value);
 std::string get_sorted_header_keys(const struct curl_slist* list);
 std::string get_canonical_headers(const struct curl_slist* list, bool only_amz = false);
+std::string get_header_value(const struct curl_slist* list, const std::string &key);
 bool MakeUrlResource(const char* realpath, std::string& resourcepath, std::string& url);
 std::string prepare_url(const char* url);
 bool get_object_sse_type(const char* path, sse_type_t& ssetype, std::string& ssevalue);   // implement in s3fs.cpp

src/fdcache.cpp

@@ -30,7 +30,6 @@
 #include <syslog.h>
 #include <errno.h>
 #include <string.h>
-#include <assert.h>
 #include <dirent.h>
 #include <curl/curl.h>
 #include <string>
@@ -52,7 +51,7 @@ using namespace std;
 //------------------------------------------------
 // Symbols
 //------------------------------------------------
-#define MAX_MULTIPART_CNT   10000                   // S3 multipart max count
+static const int MAX_MULTIPART_CNT = 10 * 1000;  // S3 multipart max count
 
 //
 // For cache directory top path
@@ -89,7 +88,7 @@ bool CacheFileStat::MakeCacheFileStatPath(const char* path, string& sfile_path,
   return true;
 }
 
-bool CacheFileStat::CheckCacheFileStatTopDir(void)
+bool CacheFileStat::CheckCacheFileStatTopDir()
 {
   if(!FdManager::IsCacheDir()){
     return true;
@@ -129,7 +128,7 @@ bool CacheFileStat::DeleteCacheFileStat(const char* path)
 // If remove stat file directory, it should do before removing
 // file cache directory.
 //
-bool CacheFileStat::DeleteCacheFileStatDirectory(void)
+bool CacheFileStat::DeleteCacheFileStatDirectory()
 {
   string top_path = FdManager::GetCacheDir();
 
@@ -175,9 +174,9 @@ bool CacheFileStat::SetPath(const char* tpath, bool is_open)
   return Open();
 }
 
-bool CacheFileStat::Open(void)
+bool CacheFileStat::Open()
 {
-  if(0 == path.size()){
+  if(path.empty()){
     return false;
   }
   if(-1 != fd){
@@ -215,7 +214,7 @@ bool CacheFileStat::Open(void)
   return true;
 }
 
-bool CacheFileStat::Release(void)
+bool CacheFileStat::Release()
 {
   if(-1 == fd){
     // already release
@@ -258,7 +257,7 @@ PageList::~PageList()
   Clear();
 }
 
-void PageList::Clear(void)
+void PageList::Clear()
 {
   PageList::FreeList(pages);
 }
@@ -271,7 +270,7 @@ bool PageList::Init(size_t size, bool is_loaded)
   return true;
 }
 
-size_t PageList::Size(void) const
+size_t PageList::Size() const
 {
   if(pages.empty()){
     return 0;
@@ -280,7 +279,7 @@ size_t PageList::Size(void) const
   return static_cast<size_t>((*riter)->next());
 }
 
-bool PageList::Compress(void)
+bool PageList::Compress()
 {
   bool is_first       = true;
   bool is_last_loaded = false;
@@ -504,7 +503,7 @@ bool PageList::Serialize(CacheFileStat& file, bool is_output)
     //
     // put to file
     //
-    stringstream ssall;
+    ostringstream ssall;
     ssall << Size();
 
     for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ++iter){
@@ -544,8 +543,8 @@ bool PageList::Serialize(CacheFileStat& file, bool is_output)
       free(ptmp);
       return false;
     }
-    string       oneline;
-    stringstream ssall(ptmp);
+    string        oneline;
+    istringstream ssall(ptmp);
 
     // loaded
     Clear();
@@ -561,8 +560,8 @@ bool PageList::Serialize(CacheFileStat& file, bool is_output)
     // load each part
     bool is_err = false;
     while(getline(ssall, oneline, '\n')){
-      string       part;
-      stringstream ssparts(oneline);
+      string        part;
+      istringstream ssparts(oneline);
       // offset
       if(!getline(ssparts, part, ':')){
         is_err = true;
@@ -601,7 +600,7 @@ bool PageList::Serialize(CacheFileStat& file, bool is_output)
   return true;
 }
 
-void PageList::Dump(void)
+void PageList::Dump()
 {
   int cnt = 0;
 
@@ -661,12 +660,12 @@ FdEntity::~FdEntity()
   }
 }
 
-void FdEntity::Clear(void)
+void FdEntity::Clear()
 {
   AutoLock auto_lock(&fdent_lock);
 
   if(-1 != fd){
-    if(0 != cachepath.size()){
+    if(!cachepath.empty()){
       CacheFileStat cfstat(path.c_str());
       if(!pagelist.Serialize(cfstat, true)){
         S3FS_PRN_WARN("failed to save cache stat file(%s).", path.c_str());
@@ -692,7 +691,7 @@ void FdEntity::Clear(void)
   is_modify     = false;
 }
 
-void FdEntity::Close(void)
+void FdEntity::Close()
 {
   S3FS_PRN_DBG("[path=%s][fd=%d][refcnt=%d]", path.c_str(), fd, (-1 != fd ? refcnt - 1 : refcnt));
 
@@ -701,9 +700,12 @@ void FdEntity::Close(void)
 
     if(0 < refcnt){
       refcnt--;
+    }else{
+      S3FS_PRN_EXIT("reference count underflow");
+      abort();
     }
     if(0 == refcnt){
-      if(0 != cachepath.size()){
+      if(!cachepath.empty()){
         CacheFileStat cfstat(path.c_str());
         if(!pagelist.Serialize(cfstat, true)){
           S3FS_PRN_WARN("failed to save cache stat file(%s).", path.c_str());
@@ -725,15 +727,12 @@ void FdEntity::Close(void)
   }
 }
 
-int FdEntity::Dup(bool no_fd_lock_wait)
+int FdEntity::Dup()
 {
   S3FS_PRN_DBG("[path=%s][fd=%d][refcnt=%d]", path.c_str(), fd, (-1 != fd ? refcnt + 1 : refcnt));
 
   if(-1 != fd){
-    AutoLock auto_lock(&fdent_lock, no_fd_lock_wait);
-    if (!auto_lock.isLockAcquired()) {
-      return -1;
-    }
+    AutoLock auto_lock(&fdent_lock);
     refcnt++;
   }
   return fd;
@@ -742,7 +741,7 @@ int FdEntity::Dup(bool no_fd_lock_wait)
 //
 // Open mirror file which is linked cache file.
 //
-int FdEntity::OpenMirrorFile(void)
+int FdEntity::OpenMirrorFile()
 {
   if(cachepath.empty()){
     S3FS_PRN_ERR("cache path is empty, why come here");
@@ -756,20 +755,35 @@ int FdEntity::OpenMirrorFile(void)
     return -EIO;
   }
 
-  // make mirror file path
-  char szfile[NAME_MAX + 1];
-  if(NULL == tmpnam(szfile)){
-    S3FS_PRN_ERR("could not get temporary file name.");
-    return -EIO;
+  // create seed generating mirror file name
+  unsigned int seed = static_cast<unsigned int>(time(NULL));
+  int urandom_fd;
+  if(-1 != (urandom_fd = open("/dev/urandom", O_RDONLY))){
+    unsigned int rand_data;
+    if(sizeof(rand_data) == read(urandom_fd, &rand_data, sizeof(rand_data))){
+      seed ^= rand_data;
+    }
+    close(urandom_fd);
   }
-  char* ppos = strrchr(szfile, '/');
-  ++ppos;
-  mirrorpath = bupdir + "/" + ppos;
 
-  // link mirror file to cache file
-  if(-1 == link(cachepath.c_str(), mirrorpath.c_str())){
-    S3FS_PRN_ERR("could not link mirror file(%s) to cache file(%s) by errno(%d).", mirrorpath.c_str(), cachepath.c_str(), errno);
-    return -errno;
+  // try to link mirror file
+  while(true){
+    // make random(temp) file path
+    // (do not care for threading, because allowed any value returned.)
+    //
+    char         szfile[NAME_MAX + 1];
+    sprintf(szfile, "%x.tmp", rand_r(&seed));
+    mirrorpath = bupdir + "/" + szfile;
+
+    // link mirror file to cache file
+    if(0 == link(cachepath.c_str(), mirrorpath.c_str())){
+      break;
+    }
+    if(EEXIST != errno){
+      S3FS_PRN_ERR("could not link mirror file(%s) to cache file(%s) by errno(%d).", mirrorpath.c_str(), cachepath.c_str(), errno);
+      return -errno;
+    }
+    ++seed;
   }
 
   // open mirror file
@@ -781,31 +795,36 @@ int FdEntity::OpenMirrorFile(void)
   return mirrorfd;
 }
 
-// [NOTE]
-// This method does not lock fdent_lock, because FdManager::fd_manager_lock
-// is locked before calling.
-//
 int FdEntity::Open(headers_t* pmeta, ssize_t size, time_t time, bool no_fd_lock_wait)
 {
   S3FS_PRN_DBG("[path=%s][fd=%d][size=%jd][time=%jd]", path.c_str(), fd, (intmax_t)size, (intmax_t)time);
 
+  AutoLock auto_lock(&fdent_lock, no_fd_lock_wait);
+  if (!auto_lock.isLockAcquired()) {
+    // had to wait for fd lock, return
+    return -EIO;
+  }
+
   if(-1 != fd){
     // already opened, needs to increment refcnt.
-    if (fd != Dup(no_fd_lock_wait)) {
-      // had to wait for fd lock, return
-      return -EIO;
-    }
+    Dup();
 
     // check only file size(do not need to save cfs and time.
     if(0 <= size && pagelist.Size() != static_cast<size_t>(size)){
       // truncate temporary file size
       if(-1 == ftruncate(fd, static_cast<size_t>(size))){
         S3FS_PRN_ERR("failed to truncate temporary file(%d) by errno(%d).", fd, errno);
+        if(0 < refcnt){
+          refcnt--;
+        }
         return -EIO;
       }
       // resize page list
       if(!pagelist.Resize(static_cast<size_t>(size), false)){
         S3FS_PRN_ERR("failed to truncate temporary file information(%d).", fd);
+        if(0 < refcnt){
+          refcnt--;
+        }
         return -EIO;
       }
     }
@@ -824,7 +843,7 @@ int FdEntity::Open(headers_t* pmeta, ssize_t size, time_t time, bool no_fd_lock_
   bool  need_save_csf = false;  // need to save(reset) cache stat file
   bool  is_truncate   = false;  // need to truncate
 
-  if(0 != cachepath.size()){
+  if(!cachepath.empty()){
     // using cache
 
     // open cache and cache stat file, load page info.
@@ -997,10 +1016,10 @@ bool FdEntity::OpenAndLoadAll(headers_t* pmeta, size_t* size, bool force_load)
 
 bool FdEntity::GetStats(struct stat& st)
 {
+  AutoLock auto_lock(&fdent_lock);
   if(-1 == fd){
     return false;
   }
-  AutoLock auto_lock(&fdent_lock);
 
   memset(&st, 0, sizeof(struct stat)); 
   if(-1 == fstat(fd, &st)){
@@ -1017,9 +1036,9 @@ int FdEntity::SetMtime(time_t time)
   if(-1 == time){
     return 0;
   }
-  if(-1 != fd){
-    AutoLock auto_lock(&fdent_lock);
 
+  AutoLock auto_lock(&fdent_lock);
+  if(-1 != fd){
     struct timeval tv[2];
     tv[0].tv_sec = time;
     tv[0].tv_usec= 0L;
@@ -1029,7 +1048,7 @@ int FdEntity::SetMtime(time_t time)
       S3FS_PRN_ERR("futimes failed. errno(%d)", errno);
       return -errno;
     }
-  }else if(0 < cachepath.size()){
+  }else if(!cachepath.empty()){
     // not opened file yet.
     struct utimbuf n_mtime;
     n_mtime.modtime = time;
@@ -1039,17 +1058,31 @@ int FdEntity::SetMtime(time_t time)
       return -errno;
     }
   }
+  orgmeta["x-amz-meta-ctime"] = str(time);
   orgmeta["x-amz-meta-mtime"] = str(time);
 
   return 0;
 }
 
-bool FdEntity::UpdateMtime(void)
+bool FdEntity::UpdateCtime()
 {
+  AutoLock auto_lock(&fdent_lock);
   struct stat st;
   if(!GetStats(st)){
     return false;
   }
+  orgmeta["x-amz-meta-ctime"] = str(st.st_ctime);
+  return true;
+}
+
+bool FdEntity::UpdateMtime()
+{
+  AutoLock auto_lock(&fdent_lock);
+  struct stat st;
+  if(!GetStats(st)){
+    return false;
+  }
+  orgmeta["x-amz-meta-ctime"] = str(st.st_ctime);
   orgmeta["x-amz-meta-mtime"] = str(st.st_mtime);
   return true;
 }
@@ -1067,18 +1100,21 @@ bool FdEntity::GetSize(size_t& size)
 
 bool FdEntity::SetMode(mode_t mode)
 {
+  AutoLock auto_lock(&fdent_lock);
   orgmeta["x-amz-meta-mode"] = str(mode);
   return true;
 }
 
 bool FdEntity::SetUId(uid_t uid)
 {
+  AutoLock auto_lock(&fdent_lock);
   orgmeta["x-amz-meta-uid"] = str(uid);
   return true;
 }
 
 bool FdEntity::SetGId(gid_t gid)
 {
+  AutoLock auto_lock(&fdent_lock);
   orgmeta["x-amz-meta-gid"] = str(gid);
   return true;
 }
@@ -1088,6 +1124,7 @@ bool FdEntity::SetContentType(const char* path)
   if(!path){
     return false;
   }
+  AutoLock auto_lock(&fdent_lock);
   orgmeta["Content-Type"] = S3fsCurl::LookupMimeType(string(path));
   return true;
 }
@@ -1146,7 +1183,7 @@ int FdEntity::Load(off_t start, size_t size)
       size_t over_size = (*iter)->bytes - need_load_size;
 
       // download
-      if(static_cast<size_t>(2 * S3fsCurl::GetMultipartSize()) < need_load_size && !nomultipart){ // default 20MB
+      if(static_cast<size_t>(2 * S3fsCurl::GetMultipartSize()) <= need_load_size && !nomultipart){ // default 20MB
         // parallel request
         // Additional time is needed for large files
         time_t backup = 0;
@@ -1209,7 +1246,7 @@ int FdEntity::NoCacheLoadAndPost(off_t start, size_t size)
   // [NOTE]
   // This method calling means that the cache file is never used no more.
   //
-  if(0 != cachepath.size()){
+  if(!cachepath.empty()){
     // remove cache files(and cache stat file)
     FdManager::DeleteCacheFile(path.c_str());
     // cache file path does not use no more.
@@ -1362,7 +1399,7 @@ int FdEntity::NoCacheLoadAndPost(off_t start, size_t size)
 // At no disk space for caching object.
 // This method is starting multipart uploading.
 //
-int FdEntity::NoCachePreMultipartPost(void)
+int FdEntity::NoCachePreMultipartPost()
 {
   // initialize multipart upload values
   upload_id.erase();
@@ -1395,7 +1432,7 @@ int FdEntity::NoCacheMultipartPost(int tgfd, off_t start, size_t size)
 // At no disk space for caching object.
 // This method is finishing multipart uploading.
 //
-int FdEntity::NoCacheCompleteMultipartPost(void)
+int FdEntity::NoCacheCompleteMultipartPost()
 {
   if(upload_id.empty() || etaglist.empty()){
     S3FS_PRN_ERR("There is no upload id or etag list.");
@@ -1420,7 +1457,7 @@ int FdEntity::NoCacheCompleteMultipartPost(void)
 
 int FdEntity::RowFlush(const char* tpath, bool force_sync)
 {
-  int result;
+  int result = 0;
 
   S3FS_PRN_INFO3("[tpath=%s][path=%s][fd=%d]", SAFESTRPTR(tpath), path.c_str(), fd);
 
@@ -1439,10 +1476,12 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
   if(0 < restsize){
     if(0 == upload_id.length()){
       // check disk space
-      if(FdManager::IsSafeDiskSpace(NULL, restsize)){
+      if(ReserveDiskSpace(restsize)){
         // enough disk space
         // Load all uninitialized area
-        if(0 != (result = Load())){
+        result = Load();
+        FdManager::FreeReservedDiskSpace(restsize);
+        if(0 != result){
           S3FS_PRN_ERR("failed to upload all area(errno=%d)", result);
           return static_cast<ssize_t>(result);
         }
@@ -1478,6 +1517,7 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
      */
     if(pagelist.Size() > static_cast<size_t>(MAX_MULTIPART_CNT * S3fsCurl::GetMultipartSize())){
       // close f ?
+      S3FS_PRN_ERR("Part count exceeds %d.  Increase multipart size and try again.", MAX_MULTIPART_CNT);
       return -ENOTSUP;
     }
 
@@ -1545,6 +1585,32 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
   return result;
 }
 
+// [NOTICE]
+// Need to lock before calling this method.
+bool FdEntity::ReserveDiskSpace(size_t size)
+{
+  if(FdManager::get()->ReserveDiskSpace(size)){
+    return true;
+  }
+
+  if(!is_modify){
+    // try to clear all cache for this fd.
+    pagelist.Init(pagelist.Size(), false);
+    if(-1 == ftruncate(fd, 0) || -1 == ftruncate(fd, pagelist.Size())){
+      S3FS_PRN_ERR("failed to truncate temporary file(%d).", fd);
+      return false;
+    }
+
+    if(FdManager::get()->ReserveDiskSpace(size)){
+      return true;
+    }
+  }
+
+  FdManager::get()->CleanupCacheDir();
+
+  return FdManager::get()->ReserveDiskSpace(size);
+}
+
 ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)
 {
   S3FS_PRN_DBG("[path=%s][fd=%d][offset=%jd][size=%zu]", path.c_str(), fd, (intmax_t)start, size);
@@ -1552,38 +1618,16 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)
   if(-1 == fd){
     return -EBADF;
   }
-  // check if not enough disk space left BEFORE locking fd
-  if(FdManager::IsCacheDir() && !FdManager::IsSafeDiskSpace(NULL, size)){
-    FdManager::get()->CleanupCacheDir();
-  }
   AutoLock auto_lock(&fdent_lock);
 
   if(force_load){
     pagelist.SetPageLoadedStatus(start, size, false);
   }
 
-  int     result;
   ssize_t rsize;
 
   // check disk space
   if(0 < pagelist.GetTotalUnloadedPageSize(start, size)){
-    if(!FdManager::IsSafeDiskSpace(NULL, size)){
-      // [NOTE]
-      // If the area of this entity fd used can be released, try to do it.
-      // But If file data is updated, we can not even release of fd.
-      // Fundamentally, this method will fail as long as the disk capacity
-      // is not ensured.
-      //
-      if(!is_modify){
-        // try to clear all cache for this fd.
-        pagelist.Init(pagelist.Size(), false);
-        if(-1 == ftruncate(fd, 0) || -1 == ftruncate(fd, pagelist.Size())){
-          S3FS_PRN_ERR("failed to truncate temporary file(%d).", fd);
-          return -ENOSPC;
-        }
-      }
-    }
-
     // load size(for prefetch)
     size_t load_size = size;
     if(static_cast<size_t>(start + size) < pagelist.Size()){
@@ -1595,8 +1639,25 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)
         load_size = static_cast<size_t>(pagelist.Size() - start);
       }
     }
+
+    if(!ReserveDiskSpace(load_size)){
+      S3FS_PRN_WARN("could not reserve disk space for pre-fetch download");
+      load_size = size;
+      if(!ReserveDiskSpace(load_size)){
+        S3FS_PRN_ERR("could not reserve disk space for pre-fetch download");
+        return -ENOSPC;
+      }
+    }
+
     // Loading
-    if(0 < size && 0 != (result = Load(start, load_size))){
+    int result = 0;
+    if(0 < size){
+      result = Load(start, load_size);
+    }
+
+    FdManager::FreeReservedDiskSpace(load_size);
+
+    if(0 != result){
       S3FS_PRN_ERR("could not download. start(%jd), size(%zu), errno(%d)", (intmax_t)start, size, result);
       return -EIO;
     }
@@ -1633,17 +1694,21 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
     pagelist.SetPageLoadedStatus(static_cast<off_t>(pagelist.Size()), static_cast<size_t>(start) - pagelist.Size(), false);
   }
 
-  int     result;
+  int     result = 0;
   ssize_t wsize;
 
   if(0 == upload_id.length()){
     // check disk space
     size_t restsize = pagelist.GetTotalUnloadedPageSize(0, start) + size;
-    if(FdManager::IsSafeDiskSpace(NULL, restsize)){
+    if(ReserveDiskSpace(restsize)){
       // enough disk space
 
       // Load uninitialized area which starts from 0 to (start + size) before writing.
-      if(0 < start && 0 != (result = Load(0, static_cast<size_t>(start)))){
+      if(0 < start){
+        result = Load(0, static_cast<size_t>(start));
+      }
+      FdManager::FreeReservedDiskSpace(restsize);
+      if(0 != result){
         S3FS_PRN_ERR("failed to load uninitialized area before writing(errno=%d)", result);
         return static_cast<ssize_t>(result);
       }
@@ -1677,6 +1742,15 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
     pagelist.SetPageLoadedStatus(start, static_cast<size_t>(wsize), true);
   }
 
+  // Load uninitialized area which starts from (start + size) to EOF after writing.
+  if(pagelist.Size() > static_cast<size_t>(start) + size){
+    result = Load(static_cast<size_t>(start + size), pagelist.Size());
+    if(0 != result){
+      S3FS_PRN_ERR("failed to load uninitialized area after writing(errno=%d)", result);
+      return static_cast<ssize_t>(result);
+    }
+  }
+
   // check multipart uploading
   if(0 < upload_id.length()){
     mp_size += static_cast<size_t>(wsize);
@@ -1710,7 +1784,7 @@ void FdEntity::CleanupCache()
   }
 
   if (is_modify) {
-    // cache is not commited to s3, cannot cleanup
+    // cache is not committed to s3, cannot cleanup
     return;
   }
 
@@ -1741,8 +1815,9 @@ void FdEntity::CleanupCache()
 FdManager       FdManager::singleton;
 pthread_mutex_t FdManager::fd_manager_lock;
 pthread_mutex_t FdManager::cache_cleanup_lock;
+pthread_mutex_t FdManager::reserved_diskspace_lock;
 bool            FdManager::is_lock_init(false);
-string          FdManager::cache_dir("");
+string          FdManager::cache_dir;
 bool            FdManager::check_cache_dir_exist(false);
 size_t          FdManager::free_disk_space = 0;
 
@@ -1759,16 +1834,26 @@ bool FdManager::SetCacheDir(const char* dir)
   return true;
 }
 
-bool FdManager::DeleteCacheDirectory(void)
+bool FdManager::DeleteCacheDirectory()
 {
-  if(0 == FdManager::cache_dir.size()){
+  if(FdManager::cache_dir.empty()){
     return true;
   }
-  string cache_dir;
-  if(!FdManager::MakeCachePath(NULL, cache_dir, false)){
+
+  string cache_path;
+  if(!FdManager::MakeCachePath(NULL, cache_path, false)){
     return false;
   }
-  return delete_files_in_dir(cache_dir.c_str(), true);
+  if(!delete_files_in_dir(cache_path.c_str(), true)){
+    return false;
+  }
+
+  string mirror_path = FdManager::cache_dir + "/." + bucket + ".mirror";
+  if(!delete_files_in_dir(mirror_path.c_str(), true)){
+    return false;
+  }
+
+  return true;
 }
 
 int FdManager::DeleteCacheFile(const char* path)
@@ -1778,10 +1863,10 @@ int FdManager::DeleteCacheFile(const char* path)
   if(!path){
     return -EIO;
   }
-  if(0 == FdManager::cache_dir.size()){
+  if(FdManager::cache_dir.empty()){
     return 0;
   }
-  string cache_path = "";
+  string cache_path;
   if(!FdManager::MakeCachePath(path, cache_path, false)){
     return 0;
   }
@@ -1811,7 +1896,7 @@ int FdManager::DeleteCacheFile(const char* path)
 
 bool FdManager::MakeCachePath(const char* path, string& cache_path, bool is_create_dir, bool is_mirror_path)
 {
-  if(0 == FdManager::cache_dir.size()){
+  if(FdManager::cache_dir.empty()){
     cache_path = "";
     return true;
   }
@@ -1841,9 +1926,9 @@ bool FdManager::MakeCachePath(const char* path, string& cache_path, bool is_crea
   return true;
 }
 
-bool FdManager::CheckCacheTopDir(void)
+bool FdManager::CheckCacheTopDir()
 {
-  if(0 == FdManager::cache_dir.size()){
+  if(FdManager::cache_dir.empty()){
     return true;
   }
   string toppath(FdManager::cache_dir + "/" + bucket);
@@ -1868,12 +1953,12 @@ bool FdManager::SetCheckCacheDirExist(bool is_check)
   return old;
 }
 
-bool FdManager::CheckCacheDirExist(void)
+bool FdManager::CheckCacheDirExist()
 {
   if(!FdManager::check_cache_dir_exist){
     return true;
   }
-  if(0 == FdManager::cache_dir.size()){
+  if(FdManager::cache_dir.empty()){
     return true;
   }
   // check the directory
@@ -1892,27 +1977,15 @@ bool FdManager::CheckCacheDirExist(void)
 size_t FdManager::SetEnsureFreeDiskSpace(size_t size)
 {
   size_t old = FdManager::free_disk_space;
-  if(0 == size){
-    if(0 == FdManager::free_disk_space){
-      FdManager::free_disk_space = static_cast<size_t>(S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount());
-    }
-  }else{
-    if(0 == FdManager::free_disk_space){
-      FdManager::free_disk_space = max(size, static_cast<size_t>(S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount()));
-    }else{
-      if(static_cast<size_t>(S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount()) <= size){
-        FdManager::free_disk_space = size;
-      }
-    }
-  }
+  FdManager::free_disk_space = size;
   return old;
 }
 
-fsblkcnt_t FdManager::GetFreeDiskSpace(const char* path)
+uint64_t FdManager::GetFreeDiskSpace(const char* path)
 {
   struct statvfs vfsbuf;
   string         ctoppath;
-  if(0 < FdManager::cache_dir.size()){
+  if(!FdManager::cache_dir.empty()){
     ctoppath = FdManager::cache_dir + "/";
     ctoppath = get_exist_directory_path(ctoppath);	// existed directory
     if(ctoppath != "/"){
@@ -1930,12 +2003,12 @@ fsblkcnt_t FdManager::GetFreeDiskSpace(const char* path)
     S3FS_PRN_ERR("could not get vfs stat by errno(%d)", errno);
     return 0;
   }
-  return (vfsbuf.f_bavail * vfsbuf.f_bsize);
+  return (vfsbuf.f_bavail * vfsbuf.f_frsize);
 }
 
 bool FdManager::IsSafeDiskSpace(const char* path, size_t size)
 {
-  fsblkcnt_t fsize = FdManager::GetFreeDiskSpace(path);
+  uint64_t fsize = FdManager::GetFreeDiskSpace(path);
   return ((size + FdManager::GetEnsureFreeDiskSpace()) <= fsize);
 }
 
@@ -1948,13 +2021,14 @@ FdManager::FdManager()
     try{
       pthread_mutex_init(&FdManager::fd_manager_lock, NULL);
       pthread_mutex_init(&FdManager::cache_cleanup_lock, NULL);
+      pthread_mutex_init(&FdManager::reserved_diskspace_lock, NULL);
       FdManager::is_lock_init = true;
     }catch(exception& e){
       FdManager::is_lock_init = false;
       S3FS_PRN_CRIT("failed to init mutex");
     }
   }else{
-    assert(false);
+    abort();
   }
 }
 
@@ -1971,13 +2045,14 @@ FdManager::~FdManager()
       try{
         pthread_mutex_destroy(&FdManager::fd_manager_lock);
         pthread_mutex_destroy(&FdManager::cache_cleanup_lock);
+        pthread_mutex_destroy(&FdManager::reserved_diskspace_lock);
       }catch(exception& e){
         S3FS_PRN_CRIT("failed to init mutex");
       }
       FdManager::is_lock_init = false;
     }
   }else{
-    assert(false);
+    abort();
   }
 }
 
@@ -1992,6 +2067,7 @@ FdEntity* FdManager::GetFdEntity(const char* path, int existfd)
 
   fdent_map_t::iterator iter = fent.find(string(path));
   if(fent.end() != iter && (-1 == existfd || (*iter).second->GetFd() == existfd)){
+    iter->second->Dup();
     return (*iter).second;
   }
 
@@ -2000,6 +2076,7 @@ FdEntity* FdManager::GetFdEntity(const char* path, int existfd)
       if((*iter).second && (*iter).second->GetFd() == existfd){
         // found opened fd in map
         if(0 == strcmp((*iter).second->GetPath(), path)){
+          iter->second->Dup();
           return (*iter).second;
         }
         // found fd, but it is used another file(file descriptor is recycled)
@@ -2018,62 +2095,73 @@ FdEntity* FdManager::Open(const char* path, headers_t* pmeta, ssize_t size, time
   if(!path || '\0' == path[0]){
     return NULL;
   }
-  AutoLock auto_lock(&FdManager::fd_manager_lock);
+  bool close = false;
+  FdEntity* ent;
+  {
+    AutoLock auto_lock(&FdManager::fd_manager_lock);
 
-  // search in mapping by key(path)
-  fdent_map_t::iterator iter = fent.find(string(path));
+    // search in mapping by key(path)
+    fdent_map_t::iterator iter = fent.find(string(path));
 
-  if(fent.end() == iter && !force_tmpfile && !FdManager::IsCacheDir()){
-    // If the cache directory is not specified, s3fs opens a temporary file
-    // when the file is opened.
-    // Then if it could not find a entity in map for the file, s3fs should
-    // search a entity in all which opened the temporary file.
-    //
-    for(iter = fent.begin(); iter != fent.end(); ++iter){
-      if((*iter).second && (*iter).second->IsOpen() && 0 == strcmp((*iter).second->GetPath(), path)){
-        break;      // found opened fd in mapping
+    if(fent.end() == iter && !force_tmpfile && !FdManager::IsCacheDir()){
+      // If the cache directory is not specified, s3fs opens a temporary file
+      // when the file is opened.
+      // Then if it could not find a entity in map for the file, s3fs should
+      // search a entity in all which opened the temporary file.
+      //
+      for(iter = fent.begin(); iter != fent.end(); ++iter){
+        if((*iter).second && (*iter).second->IsOpen() && 0 == strcmp((*iter).second->GetPath(), path)){
+          break;      // found opened fd in mapping
+        }
       }
     }
-  }
 
-  FdEntity* ent;
-  if(fent.end() != iter){
-    // found
-    ent = (*iter).second;
+    if(fent.end() != iter){
+      // found
+      ent = (*iter).second;
+      ent->Dup();
+      close = true;
 
-  }else if(is_create){
-    // not found
-    string cache_path = "";
-    if(!force_tmpfile && !FdManager::MakeCachePath(path, cache_path, true)){
-      S3FS_PRN_ERR("failed to make cache path for object(%s).", path);
+    }else if(is_create){
+      // not found
+      string cache_path;
+      if(!force_tmpfile && !FdManager::MakeCachePath(path, cache_path, true)){
+        S3FS_PRN_ERR("failed to make cache path for object(%s).", path);
+        return NULL;
+      }
+      // make new obj
+      ent = new FdEntity(path, cache_path.c_str());
+
+      if(!cache_path.empty()){
+        // using cache
+        fent[string(path)] = ent;
+      }else{
+        // not using cache, so the key of fdentity is set not really existing path.
+        // (but not strictly unexisting path.)
+        //
+        // [NOTE]
+        // The reason why this process here, please look at the definition of the
+        // comments of NOCACHE_PATH_PREFIX_FORM symbol.
+        //
+        string tmppath;
+        FdManager::MakeRandomTempPath(path, tmppath);
+        fent[tmppath] = ent;
+      }
+    }else{
       return NULL;
     }
-    // make new obj
-    ent = new FdEntity(path, cache_path.c_str());
-
-    if(0 < cache_path.size()){
-      // using cache
-      fent[string(path)] = ent;
-    }else{
-      // not using cache, so the key of fdentity is set not really existing path.
-      // (but not strictly unexisting path.)
-      //
-      // [NOTE]
-      // The reason why this process here, please look at the definition of the
-      // comments of NOCACHE_PATH_PREFIX_FORM symbol.
-      //
-      string tmppath("");
-      FdManager::MakeRandomTempPath(path, tmppath);
-      fent[tmppath] = ent;
-    }
-  }else{
-    return NULL;
   }
 
   // open
   if(0 != ent->Open(pmeta, size, time, no_fd_lock_wait)){
+    if(close){
+      ent->Close();
+    }
     return NULL;
   }
+  if(close){
+    ent->Close();
+  }
   return ent;
 }
 
@@ -2107,6 +2195,7 @@ FdEntity* FdManager::ExistOpen(const char* path, int existfd, bool ignore_existf
 
 void FdManager::Rename(const std::string &from, const std::string &to)
 {
+  AutoLock auto_lock(&FdManager::fd_manager_lock);
   fdent_map_t::iterator iter = fent.find(from);
   if(fent.end() != iter){
     // found
@@ -2159,7 +2248,7 @@ bool FdManager::ChangeEntityToTempPath(FdEntity* ent, const char* path)
     if((*iter).second == ent){
       fent.erase(iter++);
 
-      string tmppath("");
+      string tmppath;
       FdManager::MakeRandomTempPath(path, tmppath);
       fent[tmppath] = ent;
     }else{
@@ -2171,17 +2260,22 @@ bool FdManager::ChangeEntityToTempPath(FdEntity* ent, const char* path)
 
 void FdManager::CleanupCacheDir()
 {
-  if (!FdManager::IsCacheDir()) {
+  S3FS_PRN_INFO("cache cleanup requested");
+
+  if(!FdManager::IsCacheDir()){
     return;
   }
 
-  AutoLock auto_lock(&FdManager::cache_cleanup_lock, true);
+  AutoLock auto_lock_no_wait(&FdManager::cache_cleanup_lock, true);
 
-  if (!auto_lock.isLockAcquired()) {
-    return;
+  if(auto_lock_no_wait.isLockAcquired()){
+    S3FS_PRN_INFO("cache cleanup started");
+    CleanupCacheDirInternal("");
+    S3FS_PRN_INFO("cache cleanup ended");
+  }else{
+    // wait for other thread to finish cache cleanup
+    AutoLock auto_lock(&FdManager::cache_cleanup_lock);
   }
-
-  CleanupCacheDirInternal("");
 }
 
 void FdManager::CleanupCacheDirInternal(const std::string &path)
@@ -2214,16 +2308,38 @@ void FdManager::CleanupCacheDirInternal(const std::string &path)
     }else{
       FdEntity* ent;
       if(NULL == (ent = FdManager::get()->Open(next_path.c_str(), NULL, -1, -1, false, true, true))){
+        S3FS_PRN_DBG("skipping locked file: %s", next_path.c_str());
         continue;
       }
 
-      ent->CleanupCache();
+      if(ent->IsMultiOpened()){
+        S3FS_PRN_DBG("skipping opened file: %s", next_path.c_str());
+      }else{
+        ent->CleanupCache();
+        S3FS_PRN_DBG("cleaned up: %s", next_path.c_str());
+      }
       Close(ent);
     }
   }
   closedir(dp);
 }
 
+bool FdManager::ReserveDiskSpace(size_t size)
+{
+  AutoLock auto_lock(&FdManager::reserved_diskspace_lock);
+  if(IsSafeDiskSpace(NULL, size)){
+    free_disk_space += size;
+    return true;
+  }
+  return false;
+}
+
+void FdManager::FreeReservedDiskSpace(size_t size)
+{
+  AutoLock auto_lock(&FdManager::reserved_diskspace_lock);
+  free_disk_space -= size;
+}
+
 /*
 * Local variables:
 * tab-width: 4

src/fdcache.h

@@ -144,9 +144,10 @@ class FdEntity
 
     void Close(void);
     bool IsOpen(void) const { return (-1 != fd); }
+    bool IsMultiOpened(void) const { return refcnt > 1; }
     int Open(headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool no_fd_lock_wait = false);
     bool OpenAndLoadAll(headers_t* pmeta = NULL, size_t* size = NULL, bool force_load = false);
-    int Dup(bool no_fd_lock_wait = false);
+    int Dup();
 
     const char* GetPath(void) const { return path.c_str(); }
     void SetPath(const std::string &newpath) { path = newpath; }
@@ -154,6 +155,7 @@ class FdEntity
 
     bool GetStats(struct stat& st);
     int SetMtime(time_t time);
+    bool UpdateCtime(void);
     bool UpdateMtime(void);
     bool GetSize(size_t& size);
     bool SetMode(mode_t mode);
@@ -173,6 +175,7 @@ class FdEntity
     ssize_t Read(char* bytes, off_t start, size_t size, bool force_load = false);
     ssize_t Write(const char* bytes, off_t start, size_t size);
 
+    bool ReserveDiskSpace(size_t size);
     void CleanupCache();
 };
 typedef std::map<std::string, class FdEntity*> fdent_map_t;   // key=path, value=FdEntity*
@@ -186,6 +189,7 @@ class FdManager
     static FdManager       singleton;
     static pthread_mutex_t fd_manager_lock;
     static pthread_mutex_t cache_cleanup_lock;
+    static pthread_mutex_t reserved_diskspace_lock;
     static bool            is_lock_init;
     static std::string     cache_dir;
     static bool            check_cache_dir_exist;
@@ -194,7 +198,7 @@ class FdManager
     fdent_map_t            fent;
 
   private:
-    static fsblkcnt_t GetFreeDiskSpace(const char* path);
+    static uint64_t GetFreeDiskSpace(const char* path);
     void CleanupCacheDirInternal(const std::string &path = "");
 
   public:
@@ -217,9 +221,11 @@ class FdManager
 
     static size_t GetEnsureFreeDiskSpace(void) { return FdManager::free_disk_space; }
     static size_t SetEnsureFreeDiskSpace(size_t size);
-    static size_t InitEnsureFreeDiskSpace(void) { return SetEnsureFreeDiskSpace(0); }
     static bool IsSafeDiskSpace(const char* path, size_t size);
+    static void FreeReservedDiskSpace(size_t size);
+    bool ReserveDiskSpace(size_t size);
 
+    // Return FdEntity associated with path, returning NULL on error.  This operation increments the reference count; callers must decrement via Close after use.
     FdEntity* GetFdEntity(const char* path, int existfd = -1);
     FdEntity* Open(const char* path, headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true, bool no_fd_lock_wait = false);
     FdEntity* ExistOpen(const char* path, int existfd = -1, bool ignore_existfd = false);

src/gnutls_auth.cpp

@@ -57,7 +57,7 @@ const char* s3fs_crypt_lib_name(void)
 
 #else	// USE_GNUTLS_NETTLE
 
-const char* s3fs_crypt_lib_name(void)
+const char* s3fs_crypt_lib_name()
 {
   static const char version[] = "GnuTLS(gcrypt)";
 
@@ -69,15 +69,20 @@ const char* s3fs_crypt_lib_name(void)
 //-------------------------------------------------------------------
 // Utility Function for global init
 //-------------------------------------------------------------------
-bool s3fs_init_global_ssl(void)
+bool s3fs_init_global_ssl()
 {
   if(GNUTLS_E_SUCCESS != gnutls_global_init()){
     return false;
   }
+#ifndef USE_GNUTLS_NETTLE
+  if(NULL == gcry_check_version(NULL)){
+    return false;
+  }
+#endif	// USE_GNUTLS_NETTLE
   return true;
 }
 
-bool s3fs_destroy_global_ssl(void)
+bool s3fs_destroy_global_ssl()
 {
   gnutls_global_deinit();
   return true;
@@ -86,12 +91,12 @@ bool s3fs_destroy_global_ssl(void)
 //-------------------------------------------------------------------
 // Utility Function for crypt lock
 //-------------------------------------------------------------------
-bool s3fs_init_crypt_mutex(void)
+bool s3fs_init_crypt_mutex()
 {
   return true;
 }
 
-bool s3fs_destroy_crypt_mutex(void)
+bool s3fs_destroy_crypt_mutex()
 {
   return true;
 }
@@ -107,7 +112,7 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
     return false;
   }
 
-  if(NULL == (*digest = (unsigned char*)malloc(SHA1_DIGEST_SIZE))){
+  if(NULL == (*digest = reinterpret_cast<unsigned char*>(malloc(SHA1_DIGEST_SIZE)))){
     return false;
   }
 
@@ -126,7 +131,7 @@ bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, siz
     return false;
   }
 
-  if(NULL == (*digest = (unsigned char*)malloc(SHA256_DIGEST_SIZE))){
+  if(NULL == (*digest = reinterpret_cast<unsigned char*>(malloc(SHA256_DIGEST_SIZE)))){
     return false;
   }
 
@@ -150,7 +155,7 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
   if(0 == (*digestlen = gnutls_hmac_get_len(GNUTLS_MAC_SHA1))){
     return false;
   }
-  if(NULL == (*digest = (unsigned char*)malloc(*digestlen + 1))){
+  if(NULL == (*digest = reinterpret_cast<unsigned char*>(malloc(*digestlen + 1)))){
     return false;
   }
   if(0 > gnutls_hmac_fast(GNUTLS_MAC_SHA1, key, keylen, data, datalen, *digest)){
@@ -170,7 +175,7 @@ bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, siz
   if(0 == (*digestlen = gnutls_hmac_get_len(GNUTLS_MAC_SHA256))){
     return false;
   }
-  if(NULL == (*digest = (unsigned char*)malloc(*digestlen + 1))){
+  if(NULL == (*digest = reinterpret_cast<unsigned char*>(malloc(*digestlen + 1)))){
     return false;
   }
   if(0 > gnutls_hmac_fast(GNUTLS_MAC_SHA256, key, keylen, data, datalen, *digest)){
@@ -186,11 +191,9 @@ bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, siz
 //-------------------------------------------------------------------
 // Utility Function for MD5
 //-------------------------------------------------------------------
-#define MD5_DIGEST_LENGTH     16
-
-size_t get_md5_digest_length(void)
+size_t get_md5_digest_length()
 {
-  return MD5_DIGEST_LENGTH;
+  return 16;
 }
 
 #ifdef	USE_GNUTLS_NETTLE
@@ -223,7 +226,7 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
     md5_update(&ctx_md5, bytes, buf);
     memset(buf, 0, 512);
   }
-  if(NULL == (result = (unsigned char*)malloc(get_md5_digest_length()))){
+  if(NULL == (result = reinterpret_cast<unsigned char*>(malloc(get_md5_digest_length())))){
     return NULL;
   }
   md5_digest(&ctx_md5, get_md5_digest_length(), result);
@@ -274,12 +277,14 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
     }else if(-1 == bytes){
       // error
       S3FS_PRN_ERR("file read error(%d)", errno);
+      gcry_md_close(ctx_md5);
       return NULL;
     }
     gcry_md_write(ctx_md5, buf, bytes);
     memset(buf, 0, 512);
   }
-  if(NULL == (result = (unsigned char*)malloc(get_md5_digest_length()))){
+  if(NULL == (result = reinterpret_cast<unsigned char*>(malloc(get_md5_digest_length())))){
+    gcry_md_close(ctx_md5);
     return NULL;
   }
   memcpy(result, gcry_md_read(ctx_md5, 0), get_md5_digest_length());
@@ -298,11 +303,9 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
 //-------------------------------------------------------------------
 // Utility Function for SHA256
 //-------------------------------------------------------------------
-#define SHA256_DIGEST_LENGTH     32
-
-size_t get_sha256_digest_length(void)
+size_t get_sha256_digest_length()
 {
-  return SHA256_DIGEST_LENGTH;
+  return 32;
 }
 
 #ifdef	USE_GNUTLS_NETTLE
@@ -350,7 +353,7 @@ unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
     sha256_update(&ctx_sha256, bytes, buf);
     memset(buf, 0, 512);
   }
-  if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
+  if(NULL == (result = reinterpret_cast<unsigned char*>(malloc(get_sha256_digest_length())))){
     return NULL;
   }
   sha256_digest(&ctx_sha256, get_sha256_digest_length(), result);
@@ -367,8 +370,8 @@ unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
 
 bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen)
 {
-  (*digestlen) = static_cast<unsigned int>(get_sha256_digest_length());
-  if(NULL == ((*digest) = reinterpret_cast<unsigned char*>(malloc(*digestlen)))){
+  size_t len = (*digestlen) = static_cast<unsigned int>(get_sha256_digest_length());
+  if(NULL == ((*digest) = reinterpret_cast<unsigned char*>(malloc(len)))){
     return false;
   }
 
@@ -422,12 +425,14 @@ unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
     }else if(-1 == bytes){
       // error
       S3FS_PRN_ERR("file read error(%d)", errno);
+      gcry_md_close(ctx_sha256);
       return NULL;
     }
     gcry_md_write(ctx_sha256, buf, bytes);
     memset(buf, 0, 512);
   }
-  if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
+  if(NULL == (result = reinterpret_cast<unsigned char*>(malloc(get_sha256_digest_length())))){
+    gcry_md_close(ctx_sha256);
     return NULL;
   }
   memcpy(result, gcry_md_read(ctx_sha256, 0), get_sha256_digest_length());

src/nss_auth.cpp

@@ -42,7 +42,7 @@ using namespace std;
 //-------------------------------------------------------------------
 // Utility Function for version
 //-------------------------------------------------------------------
-const char* s3fs_crypt_lib_name(void)
+const char* s3fs_crypt_lib_name()
 {
   static const char version[] = "NSS";
 
@@ -52,14 +52,18 @@ const char* s3fs_crypt_lib_name(void)
 //-------------------------------------------------------------------
 // Utility Function for global init
 //-------------------------------------------------------------------
-bool s3fs_init_global_ssl(void)
+bool s3fs_init_global_ssl()
 {
-  NSS_Init(NULL);
-  NSS_NoDB_Init(NULL);
+  PR_Init(PR_USER_THREAD, PR_PRIORITY_NORMAL, 0);
+
+  if(SECSuccess != NSS_NoDB_Init(NULL)){
+    S3FS_PRN_ERR("Failed NSS_NoDB_Init call.");
+    return false;
+  }
   return true;
 }
 
-bool s3fs_destroy_global_ssl(void)
+bool s3fs_destroy_global_ssl()
 {
   NSS_Shutdown();
   PL_ArenaFinish();
@@ -70,12 +74,12 @@ bool s3fs_destroy_global_ssl(void)
 //-------------------------------------------------------------------
 // Utility Function for crypt lock
 //-------------------------------------------------------------------
-bool s3fs_init_crypt_mutex(void)
+bool s3fs_init_crypt_mutex()
 {
   return true;
 }
 
-bool s3fs_destroy_crypt_mutex(void)
+bool s3fs_destroy_crypt_mutex()
 {
   return true;
 }
@@ -92,7 +96,6 @@ static bool s3fs_HMAC_RAW(const void* key, size_t keylen, const unsigned char* d
   PK11SlotInfo* Slot;
   PK11SymKey*   pKey;
   PK11Context*  Context;
-  SECStatus     SecStatus;
   unsigned char tmpdigest[64];
   SECItem       KeySecItem   = {siBuffer, reinterpret_cast<unsigned char*>(const_cast<void*>(key)), static_cast<unsigned int>(keylen)};
   SECItem       NullSecItem  = {siBuffer, NULL, 0};
@@ -111,9 +114,9 @@ static bool s3fs_HMAC_RAW(const void* key, size_t keylen, const unsigned char* d
   }
 
   *digestlen = 0;
-  if(SECSuccess != (SecStatus = PK11_DigestBegin(Context)) ||
-     SECSuccess != (SecStatus = PK11_DigestOp(Context, data, datalen)) ||
-     SECSuccess != (SecStatus = PK11_DigestFinal(Context, tmpdigest, digestlen, sizeof(tmpdigest))) )
+  if(SECSuccess != PK11_DigestBegin(Context) ||
+     SECSuccess != PK11_DigestOp(Context, data, datalen) ||
+     SECSuccess != PK11_DigestFinal(Context, tmpdigest, digestlen, sizeof(tmpdigest)) )
   {
     PK11_DestroyContext(Context, PR_TRUE);
     PK11_FreeSymKey(pKey);
@@ -124,7 +127,7 @@ static bool s3fs_HMAC_RAW(const void* key, size_t keylen, const unsigned char* d
   PK11_FreeSymKey(pKey);
   PK11_FreeSlot(Slot);
 
-  if(NULL == (*digest = (unsigned char*)malloc(*digestlen))){
+  if(NULL == (*digest = reinterpret_cast<unsigned char*>(malloc(*digestlen)))){
     return false;
   }
   memcpy(*digest, tmpdigest, *digestlen);
@@ -145,7 +148,7 @@ bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, siz
 //-------------------------------------------------------------------
 // Utility Function for MD5
 //-------------------------------------------------------------------
-size_t get_md5_digest_length(void)
+size_t get_md5_digest_length()
 {
   return MD5_LENGTH;
 }
@@ -183,12 +186,13 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
     }else if(-1 == bytes){
       // error
       S3FS_PRN_ERR("file read error(%d)", errno);
+      PK11_DestroyContext(md5ctx, PR_TRUE);
       return NULL;
     }
     PK11_DigestOp(md5ctx, buf, bytes);
     memset(buf, 0, 512);
   }
-  if(NULL == (result = (unsigned char*)malloc(get_md5_digest_length()))){
+  if(NULL == (result = reinterpret_cast<unsigned char*>(malloc(get_md5_digest_length())))){
     PK11_DestroyContext(md5ctx, PR_TRUE);
     return NULL;
   }
@@ -206,7 +210,7 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
 //-------------------------------------------------------------------
 // Utility Function for SHA256
 //-------------------------------------------------------------------
-size_t get_sha256_digest_length(void)
+size_t get_sha256_digest_length()
 {
   return SHA256_LENGTH;
 }
@@ -269,7 +273,7 @@ unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
     PK11_DigestOp(sha256ctx, buf, bytes);
     memset(buf, 0, 512);
   }
-  if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
+  if(NULL == (result = reinterpret_cast<unsigned char*>(malloc(get_sha256_digest_length())))){
     PK11_DestroyContext(sha256ctx, PR_TRUE);
     return NULL;
   }

src/openssl_auth.cpp

@@ -46,7 +46,7 @@ using namespace std;
 //-------------------------------------------------------------------
 // Utility Function for version
 //-------------------------------------------------------------------
-const char* s3fs_crypt_lib_name(void)
+const char* s3fs_crypt_lib_name()
 {
   static const char version[] = "OpenSSL";
 
@@ -56,7 +56,7 @@ const char* s3fs_crypt_lib_name(void)
 //-------------------------------------------------------------------
 // Utility Function for global init
 //-------------------------------------------------------------------
-bool s3fs_init_global_ssl(void)
+bool s3fs_init_global_ssl()
 {
   ERR_load_crypto_strings();
   ERR_load_BIO_strings();
@@ -64,7 +64,7 @@ bool s3fs_init_global_ssl(void)
   return true;
 }
 
-bool s3fs_destroy_global_ssl(void)
+bool s3fs_destroy_global_ssl()
 {
   EVP_cleanup();
   ERR_free_strings();
@@ -93,7 +93,7 @@ static void s3fs_crypt_mutex_lock(int mode, int pos, const char* file, int line)
   }
 }
 
-static unsigned long s3fs_crypt_get_threadid(void)
+static unsigned long s3fs_crypt_get_threadid()
 {
   // For FreeBSD etc, some system's pthread_t is structure pointer.
   // Then we use cast like C style(not C++) instead of ifdef.
@@ -131,7 +131,7 @@ static void s3fs_destroy_dyn_crypt_mutex(struct CRYPTO_dynlock_value* dyndata, c
   }
 }
 
-bool s3fs_init_crypt_mutex(void)
+bool s3fs_init_crypt_mutex()
 {
   if(s3fs_crypt_mutex){
     S3FS_PRN_DBG("s3fs_crypt_mutex is not NULL, destroy it.");
@@ -158,7 +158,7 @@ bool s3fs_init_crypt_mutex(void)
   return true;
 }
 
-bool s3fs_destroy_crypt_mutex(void)
+bool s3fs_destroy_crypt_mutex()
 {
   if(!s3fs_crypt_mutex){
     return true;
@@ -214,7 +214,7 @@ bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, siz
 //-------------------------------------------------------------------
 // Utility Function for MD5
 //-------------------------------------------------------------------
-size_t get_md5_digest_length(void)
+size_t get_md5_digest_length()
 {
   return MD5_DIGEST_LENGTH;
 }
@@ -273,7 +273,7 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
 //-------------------------------------------------------------------
 // Utility Function for SHA256
 //-------------------------------------------------------------------
-size_t get_sha256_digest_length(void)
+size_t get_sha256_digest_length()
 {
   return SHA256_DIGEST_LENGTH;
 }

src/psemaphore.h

@@ -0,0 +1,75 @@
+/*
+ * s3fs - FUSE-based file system backed by Amazon S3
+ *
+ * Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef S3FS_SEMAPHORE_H_
+#define S3FS_SEMAPHORE_H_
+
+// portability wrapper for sem_t since macOS does not implement it
+
+#ifdef __APPLE__
+
+#include <dispatch/dispatch.h>
+
+class Semaphore
+{
+  public:
+    explicit Semaphore(int value) : value(value), sem(dispatch_semaphore_create(value)) {}
+    ~Semaphore() {
+      // macOS cannot destroy a semaphore with posts less than the initializer
+      for(int i = 0; i < get_value(); ++i){
+        post();
+      }
+      dispatch_release(sem);
+    }
+    void wait() { dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); }
+    void post() { dispatch_semaphore_signal(sem); }
+    int get_value() const { return value; }
+  private:
+    const int value;
+    dispatch_semaphore_t sem;
+};
+
+#else
+
+#include <errno.h>
+#include <semaphore.h>
+
+class Semaphore
+{
+  public:
+    explicit Semaphore(int value) : value(value) { sem_init(&mutex, 0, value); }
+    ~Semaphore() { sem_destroy(&mutex); }
+    void wait()
+    {
+      int r;
+      do {
+        r = sem_wait(&mutex);
+      } while (r == -1 && errno == EINTR);
+    }
+    void post() { sem_post(&mutex); }
+    int get_value() const { return value; }
+  private:
+    const int value;
+    sem_t mutex;
+};
+
+#endif
+
+#endif // S3FS_SEMAPHORE_H_

src/s3fs.h

@@ -21,7 +21,8 @@
 #define S3FS_S3_H_
 
 #define FUSE_USE_VERSION      26
-#define FIVE_GB               5368709120LL
+
+static const int64_t FIVE_GB = 5LL * 1024LL * 1024LL * 1024LL;
 
 #include <fuse.h>
 
@@ -32,25 +33,32 @@
   } \
 }
 
+// [NOTE]
+// s3fs use many small allocated chunk in heap area for stats
+// cache and parsing xml, etc. The OS may decide that giving
+// this little memory back to the kernel will cause too much
+// overhead and delay the operation.
+// Address of gratitude, this workaround quotes a document of
+// libxml2.( http://xmlsoft.org/xmlmem.html )
 //
-// s3fs use many small allocated chunk in heap area for
-// stats cache and parsing xml, etc. The OS may decide
-// that giving this little memory back to the kernel
-// will cause too much overhead and delay the operation.
-// So s3fs calls malloc_trim function to really get the
-// memory back. Following macros is prepared for that
-// your system does not have it.
-//
-// Address of gratitude, this workaround quotes a document
-// of libxml2.
-// http://xmlsoft.org/xmlmem.html
+// When valgrind is used to test memory leak of s3fs, a large
+// amount of chunk may be reported. You can check the memory
+// release accurately by defining the S3FS_MALLOC_TRIM flag
+// and building it. Also, when executing s3fs, you can define
+// the MMAP_THRESHOLD environment variable and check more
+// accurate memory leak.( see, man 3 free )
 //
+#ifdef S3FS_MALLOC_TRIM
 #ifdef HAVE_MALLOC_TRIM
-
 #include <malloc.h>
+#define S3FS_MALLOCTRIM(pad)    malloc_trim(pad)
+#else   // HAVE_MALLOC_TRIM
+#define S3FS_MALLOCTRIM(pad)
+#endif  // HAVE_MALLOC_TRIM
+#else   // S3FS_MALLOC_TRIM
+#define S3FS_MALLOCTRIM(pad)
+#endif  // S3FS_MALLOC_TRIM
 
-#define DISPWARN_MALLOCTRIM(str)
-#define S3FS_MALLOCTRIM(pad)          malloc_trim(pad)
 #define S3FS_XMLFREEDOC(doc) \
         { \
           xmlFreeDoc(doc); \
@@ -72,18 +80,6 @@
           S3FS_MALLOCTRIM(0); \
         }
 
-#else // HAVE_MALLOC_TRIM
-
-#define DISPWARN_MALLOCTRIM(str) \
-        fprintf(stderr, "Warning: %s without malloc_trim is possibility of the use memory increase.\n", program_name.c_str())
-#define S3FS_MALLOCTRIM(pad)
-#define S3FS_XMLFREEDOC(doc)          xmlFreeDoc(doc)
-#define S3FS_XMLFREE(ptr)             xmlFree(ptr)
-#define S3FS_XMLXPATHFREECONTEXT(ctx) xmlXPathFreeContext(ctx)
-#define S3FS_XMLXPATHFREEOBJECT(obj)  xmlXPathFreeObject(obj)
-
-#endif // HAVE_MALLOC_TRIM
-
 #endif // S3FS_S3_H_
 
 /*

src/s3fs_util.cpp

@@ -48,7 +48,7 @@ using namespace std;
 //-------------------------------------------------------------------
 // Global variables
 //-------------------------------------------------------------------
-std::string mount_prefix   = "";
+std::string mount_prefix;
 
 //-------------------------------------------------------------------
 // Utility
@@ -150,10 +150,10 @@ bool S3ObjList::insert_normalized(const char* name, const char* normalized, bool
   s3obj_t::iterator iter;
   if(objects.end() != (iter = objects.find(name))){
     // found name --> over write
-    (*iter).second.orgname.erase();
-    (*iter).second.etag.erase();
-    (*iter).second.normalname = normalized;
-    (*iter).second.is_dir     = is_dir;
+    iter->second.orgname.erase();
+    iter->second.etag.erase();
+    iter->second.normalname = normalized;
+    iter->second.is_dir     = is_dir;
   }else{
     // not found --> add new object
     s3obj_entry newobject;
@@ -259,7 +259,7 @@ bool S3ObjList::GetNameList(s3obj_list_t& list, bool OnlyNormalized, bool CutSla
     }
     string name = (*iter).first;
     if(CutSlash && 1 < name.length() && '/' == name[name.length() - 1]){
-      // only "/" string is skio this.
+      // only "/" string is skipped this.
       name = name.substr(0, name.length() - 1);
     }
     list.push_back(name);
@@ -283,7 +283,7 @@ bool S3ObjList::MakeHierarchizedList(s3obj_list_t& list, bool haveSlash)
     h_map[strtmp] = true;
 
     // check hierarchized directory
-    for(string::size_type pos = strtmp.find_last_of("/"); string::npos != pos; pos = strtmp.find_last_of("/")){
+    for(string::size_type pos = strtmp.find_last_of('/'); string::npos != pos; pos = strtmp.find_last_of('/')){
       strtmp = strtmp.substr(0, pos);
       if(0 == strtmp.length() || "/" == strtmp){
         break;
@@ -419,7 +419,6 @@ void free_mvnodes(MVNODE *head)
     free(my_head->new_path);
     free(my_head);
   }
-  return;
 }
 
 //-------------------------------------------------------------------
@@ -453,6 +452,7 @@ AutoLock::~AutoLock()
 string get_username(uid_t uid)
 {
   static size_t maxlen = 0;	// set once
+  int result;
   char* pbuf;
   struct passwd pwinfo;
   struct passwd* ppwinfo = NULL;
@@ -461,9 +461,17 @@ string get_username(uid_t uid)
   if(0 == maxlen){
     long res = sysconf(_SC_GETPW_R_SIZE_MAX);
     if(0 > res){
-      S3FS_PRN_WARN("could not get max pw length.");
-      maxlen = 0;
-      return string("");
+      // SUSv4tc1 says the following about _SC_GETGR_R_SIZE_MAX and
+      // _SC_GETPW_R_SIZE_MAX:
+      // Note that sysconf(_SC_GETGR_R_SIZE_MAX) may return -1 if
+      // there is no hard limit on the size of the buffer needed to
+      // store all the groups returned.
+      if (errno != 0){
+        S3FS_PRN_WARN("could not get max pw length.");
+        maxlen = 0;
+        return string("");
+      }
+      res = 1024; // default initial length
     }
     maxlen = res;
   }
@@ -471,12 +479,22 @@ string get_username(uid_t uid)
     S3FS_PRN_CRIT("failed to allocate memory.");
     return string("");
   }
-  // get group information
-  if(0 != getpwuid_r(uid, &pwinfo, pbuf, maxlen, &ppwinfo)){
-    S3FS_PRN_WARN("could not get pw information.");
+  // get pw information
+  while(ERANGE == (result = getpwuid_r(uid, &pwinfo, pbuf, maxlen, &ppwinfo))){
+    free(pbuf);
+    maxlen *= 2;
+    if(NULL == (pbuf = (char*)malloc(sizeof(char) * maxlen))){
+      S3FS_PRN_CRIT("failed to allocate memory.");
+      return string("");
+    }
+  }
+
+  if(0 != result){
+    S3FS_PRN_ERR("could not get pw information(%d).", result);
     free(pbuf);
     return string("");
   }
+
   // check pw
   if(NULL == ppwinfo){
     free(pbuf);
@@ -498,10 +516,18 @@ int is_uid_include_group(uid_t uid, gid_t gid)
   // make buffer
   if(0 == maxlen){
     long res = sysconf(_SC_GETGR_R_SIZE_MAX);
-    if(0 > res){
-      S3FS_PRN_ERR("could not get max name length.");
-      maxlen = 0;
-      return -ERANGE;
+    if(0 > res) {
+      // SUSv4tc1 says the following about _SC_GETGR_R_SIZE_MAX and
+      // _SC_GETPW_R_SIZE_MAX:
+      // Note that sysconf(_SC_GETGR_R_SIZE_MAX) may return -1 if
+      // there is no hard limit on the size of the buffer needed to
+      // store all the groups returned.
+      if (errno != 0) {
+        S3FS_PRN_ERR("could not get max name length.");
+        maxlen = 0;
+        return -ERANGE;
+      }
+      res = 1024; // default initial length
     }
     maxlen = res;
   }
@@ -510,8 +536,17 @@ int is_uid_include_group(uid_t uid, gid_t gid)
     return -ENOMEM;
   }
   // get group information
-  if(0 != (result = getgrgid_r(gid, &ginfo, pbuf, maxlen, &pginfo))){
-    S3FS_PRN_ERR("could not get group information.");
+  while(ERANGE == (result = getgrgid_r(gid, &ginfo, pbuf, maxlen, &pginfo))){
+    free(pbuf);
+    maxlen *= 2;
+    if(NULL == (pbuf = (char*)malloc(sizeof(char) * maxlen))){
+      S3FS_PRN_CRIT("failed to allocate memory.");
+      return -ENOMEM;
+    }
+  }
+
+  if(0 != result){
+    S3FS_PRN_ERR("could not get group information(%d).", result);
     free(pbuf);
     return -result;
   }
@@ -550,7 +585,7 @@ string mydirname(const char* path)
   return mydirname(string(path));
 }
 
-string mydirname(string path)
+string mydirname(const string& path)
 {
   return string(dirname((char*)path.c_str()));
 }
@@ -565,7 +600,7 @@ string mybasename(const char* path)
   return mybasename(string(path));
 }
 
-string mybasename(string path)
+string mybasename(const string& path)
 {
   return string(basename((char*)path.c_str()));
 }
@@ -573,9 +608,9 @@ string mybasename(string path)
 // mkdir --parents
 int mkdirp(const string& path, mode_t mode)
 {
-  string       base;
-  string       component;
-  stringstream ss(path);
+  string        base;
+  string        component;
+  istringstream ss(path);
   while (getline(ss, component, '/')) {
     base += "/" + component;
 
@@ -585,7 +620,7 @@ int mkdirp(const string& path, mode_t mode)
         return EPERM;
       }
     }else{
-      if(0 != mkdir(base.c_str(), mode)){
+      if(0 != mkdir(base.c_str(), mode) && errno != EEXIST){
         return errno;
      }
     }
@@ -596,10 +631,10 @@ int mkdirp(const string& path, mode_t mode)
 // get existed directory path
 string get_exist_directory_path(const string& path)
 {
-  string       existed("/");    // "/" is existed.
-  string       base;
-  string       component;
-  stringstream ss(path);
+  string        existed("/");    // "/" is existed.
+  string        base;
+  string        component;
+  istringstream ss(path);
   while (getline(ss, component, '/')) {
     if(base != "/"){
       base += "/";
@@ -625,7 +660,7 @@ bool check_exist_dir_permission(const char* dirpath)
   struct stat st;
   if(0 != stat(dirpath, &st)){
     if(ENOENT == errno){
-      // dir does not exitst
+      // dir does not exist
       return true;
     }
     if(EACCES == errno){
@@ -712,15 +747,29 @@ bool delete_files_in_dir(const char* dir, bool is_remove_own)
 //-------------------------------------------------------------------
 // Utility functions for convert
 //-------------------------------------------------------------------
-time_t get_mtime(const char *s)
+time_t get_mtime(const char *str)
 {
-  return static_cast<time_t>(s3fs_strtoofft(s));
+  // [NOTE]
+  // In rclone, there are cases where ns is set to x-amz-meta-mtime
+  // with floating point number. s3fs uses x-amz-meta-mtime by
+  // truncating the floating point or less (in seconds or less) to
+  // correspond to this.
+  //
+  string strmtime;
+  if(str && '\0' != *str){
+    strmtime = str;
+    string::size_type pos = strmtime.find('.', 0);
+    if(string::npos != pos){
+      strmtime = strmtime.substr(0, pos);
+    }
+  }
+  return static_cast<time_t>(s3fs_strtoofft(strmtime.c_str()));
 }
 
-time_t get_mtime(headers_t& meta, bool overcheck)
+static time_t get_time(headers_t& meta, bool overcheck, const char *header)
 {
   headers_t::const_iterator iter;
-  if(meta.end() == (iter = meta.find("x-amz-meta-mtime"))){
+  if(meta.end() == (iter = meta.find(header))){
     if(overcheck){
       return get_lastmodified(meta);
     }
@@ -729,6 +778,16 @@ time_t get_mtime(headers_t& meta, bool overcheck)
   return get_mtime((*iter).second.c_str());
 }
 
+time_t get_mtime(headers_t& meta, bool overcheck)
+{
+  return get_time(meta, overcheck, "x-amz-meta-mtime");
+}
+
+time_t get_ctime(headers_t& meta, bool overcheck)
+{
+  return get_time(meta, overcheck, "x-amz-meta-ctime");
+}
+
 off_t get_size(const char *s)
 {
   return s3fs_strtoofft(s);
@@ -756,11 +815,13 @@ mode_t get_mode(headers_t& meta, const char* path, bool checkdir, bool forcedir)
 
   if(meta.end() != (iter = meta.find("x-amz-meta-mode"))){
     mode = get_mode((*iter).second.c_str());
+  }else if(meta.end() != (iter = meta.find("x-amz-meta-permissions"))){ // for s3sync
+    mode = get_mode((*iter).second.c_str());
+    isS3sync = true;
   }else{
-    if(meta.end() != (iter = meta.find("x-amz-meta-permissions"))){ // for s3sync
-      mode = get_mode((*iter).second.c_str());
-      isS3sync = true;
-    }
+    // If another tool creates an object without permissions, default to owner
+    // read-write and group readable.
+    mode = path[strlen(path) - 1] == '/' ? 0750 : 0640;
   }
   // Checking the bitmask, if the last 3 bits are all zero then process as a regular
   // file type (S_IFDIR or S_IFREG), otherwise return mode unmodified so that S_IFIFO,
@@ -774,18 +835,19 @@ mode_t get_mode(headers_t& meta, const char* path, bool checkdir, bool forcedir)
           if(meta.end() != (iter = meta.find("Content-Type"))){
             string strConType = (*iter).second;
             // Leave just the mime type, remove any optional parameters (eg charset)
-            string::size_type pos = strConType.find(";");
+            string::size_type pos = strConType.find(';');
             if(string::npos != pos){
               strConType = strConType.substr(0, pos);
             }
-            if(strConType == "application/x-directory"){
+            if(strConType == "application/x-directory" 
+	       || strConType == "httpd/unix-directory"){ // Nextcloud uses this MIME type for directory objects when mounting bucket as external Storage
               mode |= S_IFDIR;
             }else if(path && 0 < strlen(path) && '/' == path[strlen(path) - 1]){
               if(strConType == "binary/octet-stream" || strConType == "application/octet-stream"){
                 mode |= S_IFDIR;
               }else{
                 if(complement_stat){
-                  // If complement lack stat mode, when the object has '/' charactor at end of name
+                  // If complement lack stat mode, when the object has '/' character at end of name
                   // and content type is text/plain and the object's size is 0 or 1, it should be
                   // directory.
                   off_t size = get_size(meta);
@@ -830,12 +892,13 @@ uid_t get_uid(const char *s)
 uid_t get_uid(headers_t& meta)
 {
   headers_t::const_iterator iter;
-  if(meta.end() == (iter = meta.find("x-amz-meta-uid"))){
-    if(meta.end() == (iter = meta.find("x-amz-meta-owner"))){ // for s3sync
-      return 0;
-    }
+  if(meta.end() != (iter = meta.find("x-amz-meta-uid"))){
+    return get_uid((*iter).second.c_str());
+  }else if(meta.end() != (iter = meta.find("x-amz-meta-owner"))){ // for s3sync
+    return get_uid((*iter).second.c_str());
+  }else{
+    return geteuid();
   }
-  return get_uid((*iter).second.c_str());
 }
 
 gid_t get_gid(const char *s)
@@ -846,12 +909,13 @@ gid_t get_gid(const char *s)
 gid_t get_gid(headers_t& meta)
 {
   headers_t::const_iterator iter;
-  if(meta.end() == (iter = meta.find("x-amz-meta-gid"))){
-    if(meta.end() == (iter = meta.find("x-amz-meta-group"))){ // for s3sync
-      return 0;
-    }
+  if(meta.end() != (iter = meta.find("x-amz-meta-gid"))){
+    return get_gid((*iter).second.c_str());
+  }else if(meta.end() != (iter = meta.find("x-amz-meta-group"))){ // for s3sync
+    return get_gid((*iter).second.c_str());
+  }else{
+    return getegid();
   }
-  return get_gid((*iter).second.c_str());
 }
 
 blkcnt_t get_blocks(off_t size)
@@ -928,13 +992,13 @@ bool is_need_check_obj_detail(headers_t& meta)
 //-------------------------------------------------------------------
 // Help
 //-------------------------------------------------------------------
-void show_usage (void)
+void show_usage ()
 {
   printf("Usage: %s BUCKET:[PATH] MOUNTPOINT [OPTION]...\n",
     program_name.c_str());
 }
 
-void show_help (void)
+void show_help ()
 {
   show_usage();
   printf(
@@ -949,13 +1013,14 @@ void show_help (void)
     "   umounting\n"
     "     umount mountpoint\n"
     "\n"
-    "   utility mode (remove interrupted multipart uploading objects)\n"
-    "     s3fs -u bucket\n"
-    "\n"
     "   General forms for s3fs and FUSE/mount options:\n"
     "      -o opt[,opt...]\n"
     "      -o opt [-o opt] ...\n"
     "\n"
+    "   utility mode (remove interrupted multipart uploading objects)\n"
+    "     s3fs --incomplete-mpu-list(-u) bucket\n"
+    "     s3fs --incomplete-mpu-abort[=all | =<date format>] bucket\n"
+    "\n"
     "s3fs Options:\n"
     "\n"
     "   Most s3fs options are given in the form where \"opt\" is:\n"
@@ -969,10 +1034,11 @@ void show_help (void)
     "   default_acl (default=\"private\")\n"
     "      - the default canned acl to apply to all written s3 objects,\n"
     "        e.g., private, public-read.  empty string means do not send\n"
-    "        header.  see http://aws.amazon.com/documentation/s3/ for the\n"
-    "        full list of canned acls\n"
+    "        header.  see\n"
+    "        https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl\n"
+    "        for the full list of canned acls\n"
     "\n"
-    "   retries (default=\"2\")\n"
+    "   retries (default=\"5\")\n"
     "      - number of times to retry a failed s3 transaction\n"
     "\n"
     "   use_cache (default=\"\" which means disabled)\n"
@@ -988,7 +1054,7 @@ void show_help (void)
     "\n"
     "   storage_class (default=\"standard\")\n"
     "      - store object with specified storage class.  Possible values:\n"
-    "        standard, standard_ia, and reduced_redundancy.\n"
+    "        standard, standard_ia, onezone_ia and reduced_redundancy.\n"
     "\n"
     "   use_sse (default is disable)\n"
     "      - Specify three type Amazon's Server-Site Encryption: SSE-S3,\n"
@@ -1068,13 +1134,22 @@ void show_help (void)
     "      If you specify this option for set \"Content-Encoding\" HTTP \n"
     "      header, please take care for RFC 2616.\n"
     "\n"
+    "   profile (default=\"default\")\n"
+    "      - Choose a profile from ${HOME}/.aws/credentials to authenticate\n"
+    "        against S3. Note that this format matches the AWS CLI format and\n"
+    "        differs from the s3fs passwd format.\n"
+    "\n"
     "   connect_timeout (default=\"300\" seconds)\n"
     "      - time to wait for connection before giving up\n"
     "\n"
     "   readwrite_timeout (default=\"60\" seconds)\n"
     "      - time to wait between read/write activity before giving up\n"
     "\n"
-    "   max_stat_cache_size (default=\"1000\" entries (about 4MB))\n"
+    "   list_object_max_keys (default=\"1000\")\n"
+    "      - specify the maximum number of keys returned by S3 list object\n"
+    "        API. The default is 1000. you can set this value to 1000 or more.\n"
+    "\n"
+    "   max_stat_cache_size (default=\"100,000\" entries (about 40MB))\n"
     "      - maximum number of entries in the stat cache\n"
     "\n"
     "   stat_cache_expire (default is no expire)\n"
@@ -1116,13 +1191,13 @@ void show_help (void)
     "   multipart_size (default=\"10\")\n"
     "      - part size, in MB, for each multipart request.\n"
     "\n"
-    "   ensure_diskfree (default same multipart_size value)\n"
+    "   ensure_diskfree (default 0)\n"
     "      - sets MB to ensure disk free space. s3fs makes file for\n"
     "        downloading, uploading and caching files. If the disk free\n"
     "        space is smaller than this value, s3fs do not use diskspace\n"
     "        as possible in exchange for the performance.\n"
     "\n"
-    "   singlepart_copy_limit (default=\"5120\")\n"
+    "   singlepart_copy_limit (default=\"512\")\n"
     "      - maximum size, in MB, of a single-part copy before trying \n"
     "      multipart copy.\n"
     "\n"
@@ -1143,7 +1218,7 @@ void show_help (void)
     "      error from the S3 server.\n"
     "\n"
     "   sigv2 (default is signature version 4)\n"
-    "      - sets signing AWS requests by sing Signature Version 2\n"
+    "      - sets signing AWS requests by using Signature Version 2\n"
     "\n"
     "   mp_umask (default is \"0000\")\n"
     "      - sets umask for the mount point directory.\n"
@@ -1156,7 +1231,12 @@ void show_help (void)
     "   nomultipart (disable multipart uploads)\n"
     "\n"
     "   enable_content_md5 (default is disable)\n"
-    "      - ensure data integrity during writes with MD5 hash.\n"
+    "      Allow S3 server to check data integrity of uploads via the\n"
+    "      Content-MD5 header.  This can add CPU overhead to transfers.\n"
+    "\n"
+    "   ecs\n"
+    "      - This option instructs s3fs to query the ECS container credential\n"
+    "      metadata address instead of the instance metadata address.\n"
     "\n"
     "   iam_role (default is no IAM role)\n"
     "      - This option requires the IAM role name or \"auto\". If you specify\n"
@@ -1164,6 +1244,14 @@ void show_help (void)
     "      to an instance. If you specify this option without any argument, it\n"
     "      is the same as that you have specified the \"auto\".\n"
     "\n"
+    "   ibm_iam_auth\n"
+    "      - This option instructs s3fs to use IBM IAM authentication.\n"
+    "      In this mode, the AWSAccessKey and AWSSecretKey will be used as\n"
+    "      IBM's Service-Instance-ID and APIKey, respectively.\n"
+    "\n"
+    "   ibm_iam_endpoint (default is https://iam.bluemix.net)\n"
+    "      - sets the url to use for IBM IAM authentication.\n"
+    "\n"
     "   use_xattr (default is not handling the extended attribute)\n"
     "      Enable to handle the extended attribute(xattrs).\n"
     "      If you set this option, you can use the extended attribute.\n"
@@ -1222,6 +1310,9 @@ void show_help (void)
     "        can be found on the CURL library documentation:\n"
     "        https://curl.haxx.se/docs/ssl-ciphers.html\n"
     "\n"
+    "   instance_name - The instance name of the current s3fs mountpoint.\n"
+    "        This name will be added to logging messages and user agent headers sent by s3fs.\n"
+    "\n"
     "   complement_stat (complement lack of file/directory mode)\n"
     "        s3fs complements lack of information about file/directory mode\n"
     "        if a file or a directory object does not have x-amz-meta-mode\n"
@@ -1247,6 +1338,14 @@ void show_help (void)
     "        Please use this option when the directory in the bucket is\n"
     "        only \"dir/\" object.\n"
     "\n"
+    "   use_wtf8 - support arbitrary file system encoding.\n"
+    "        S3 requires all object names to be valid utf-8. But some\n"
+    "        clients, notably Windows NFS clients, use their own encoding.\n"
+    "        This option re-encodes invalid utf-8 object names into valid\n"
+    "        utf-8 by mapping offending codes into a 'private' codepage of the\n"
+    "        Unicode set.\n"
+    "        Useful on clients not using utf-8 as their file system encoding.\n"
+    "\n"
     "FUSE/mount Options:\n"
     "\n"
     "   Most of the generic mount options described in 'man mount' are\n"
@@ -1258,6 +1357,22 @@ void show_help (void)
     "   There are many FUSE specific mount options that can be specified.\n"
     "   e.g. allow_other  See the FUSE's README for the full set.\n"
     "\n"
+    "Utility mode Options:\n"
+    "\n"
+    " -u, --incomplete-mpu-list\n"
+    "        Lists multipart incomplete objects uploaded to the specified\n"
+    "        bucket.\n"
+    " --incomplete-mpu-abort(=all or =<date format>)\n"
+    "        Delete the multipart incomplete object uploaded to the specified\n"
+    "        bucket.\n"
+    "        If \"all\" is specified for this option, all multipart incomplete\n"
+    "        objects will be deleted. If you specify no argument as an option,\n"
+    "        objects older than 24 hours(24H) will be deleted(This is the\n"
+    "        default value). You can specify an optional date format. It can\n"
+    "        be specified as year, month, day, hour, minute, second, and it is\n"
+    "        expressed as \"Y\", \"M\", \"D\", \"h\", \"m\", \"s\" respectively.\n"
+    "        For example, \"1Y6M10D12h30m30s\".\n"
+    "\n"
     "Miscellaneous Options:\n"
     "\n"
     " -h, --help        Output this help.\n"
@@ -1271,19 +1386,17 @@ void show_help (void)
     "\n"
     "s3fs home page: <https://github.com/s3fs-fuse/s3fs-fuse>\n"
   );
-  return;
 }
 
-void show_version(void)
+void show_version()
 {
   printf(
   "Amazon Simple Storage Service File System V%s(commit:%s) with %s\n"
   "Copyright (C) 2010 Randy Rizun <rrizun@gmail.com>\n"
-  "License GPL2: GNU GPL version 2 <http://gnu.org/licenses/gpl.html>\n"
+  "License GPL2: GNU GPL version 2 <https://gnu.org/licenses/gpl.html>\n"
   "This is free software: you are free to change and redistribute it.\n"
   "There is NO WARRANTY, to the extent permitted by law.\n",
   VERSION, COMMIT_HASH_VAL, s3fs_crypt_lib_name());
-  return;
 }
 
 /*

src/s3fs_util.h

@@ -109,9 +109,9 @@ std::string get_username(uid_t uid);
 int is_uid_include_group(uid_t uid, gid_t gid);
 
 std::string mydirname(const char* path);
-std::string mydirname(std::string path);
+std::string mydirname(const std::string& path);
 std::string mybasename(const char* path);
-std::string mybasename(std::string path);
+std::string mybasename(const std::string& path);
 int mkdirp(const std::string& path, mode_t mode);
 std::string get_exist_directory_path(const std::string& path);
 bool check_exist_dir_permission(const char* dirpath);
@@ -119,6 +119,7 @@ bool delete_files_in_dir(const char* dir, bool is_remove_own);
 
 time_t get_mtime(const char *s);
 time_t get_mtime(headers_t& meta, bool overcheck = true);
+time_t get_ctime(headers_t& meta, bool overcheck = true);
 off_t get_size(const char *s);
 off_t get_size(headers_t& meta);
 mode_t get_mode(const char *s);

src/string_util.cpp

@@ -22,6 +22,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <syslog.h>
+#include <time.h>
 
 #include <sstream>
 #include <string>
@@ -32,6 +33,21 @@
 
 using namespace std;
 
+template <class T> std::string str(T value) {
+  std::ostringstream s;
+  s << value;
+  return s.str();
+}
+
+template std::string str(short value);
+template std::string str(unsigned short value);
+template std::string str(int value);
+template std::string str(unsigned int value);
+template std::string str(long value);
+template std::string str(unsigned long value);
+template std::string str(long long value);
+template std::string str(unsigned long long value);
+
 static const char hexAlphabet[] = "0123456789ABCDEF";
 
 off_t s3fs_strtoofft(const char* str, bool is_base_16)
@@ -60,7 +76,7 @@ off_t s3fs_strtoofft(const char* str, bool is_base_16)
     }
     // check like isalnum and set data
     result *= (is_base_16 ? 16 : 10);
-    if('0' <= *str || '9' < *str){
+    if('0' <= *str && '9' >= *str){
       result += static_cast<off_t>(*str - '0');
     }else if(is_base_16){
       if('A' <= *str && *str <= 'F'){
@@ -105,8 +121,7 @@ string trim_right(const string &s, const string &t /* = SPACES */)
 
 string trim(const string &s, const string &t /* = SPACES */)
 {
-  string d(s);
-  return trim_left(trim_right(d, t), t);
+  return trim_left(trim_right(s, t), t);
 }
 
 /**
@@ -195,15 +210,15 @@ bool takeout_str_dquart(string& str)
   size_t pos;
 
   // '"' for start
-  if(string::npos != (pos = str.find_first_of("\""))){
+  if(string::npos != (pos = str.find_first_of('\"'))){
     str = str.substr(pos + 1);
 
     // '"' for end
-    if(string::npos == (pos = str.find_last_of("\""))){
+    if(string::npos == (pos = str.find_last_of('\"'))){
       return false;
     }
     str = str.substr(0, pos);
-    if(string::npos != str.find_first_of("\"")){
+    if(string::npos != str.find_first_of('\"')){
       return false;
     }
   }
@@ -269,6 +284,75 @@ string get_date_iso8601(time_t tm)
   return buf;
 }
 
+bool get_unixtime_from_iso8601(const char* pdate, time_t& unixtime)
+{
+  if(!pdate){
+    return false;
+  }
+
+  struct tm tm;
+  char*     prest = strptime(pdate, "%Y-%m-%dT%T", &tm);
+  if(prest == pdate){
+    // wrong format
+    return false;
+  }
+  unixtime = mktime(&tm);
+  return true;
+}
+
+//
+// Convert to unixtime from string which formatted by following:
+//   "12Y12M12D12h12m12s", "86400s", "9h30m", etc
+//
+bool convert_unixtime_from_option_arg(const char* argv, time_t& unixtime)
+{
+  if(!argv){
+    return false;
+  }
+  unixtime = 0;
+  const char* ptmp;
+  int         last_unit_type = 0;       // unit flag.
+  bool        is_last_number;
+  time_t      tmptime;
+  for(ptmp = argv, is_last_number = true, tmptime = 0; ptmp && *ptmp; ++ptmp){
+    if('0' <= *ptmp && *ptmp <= '9'){
+      tmptime        *= 10;
+      tmptime        += static_cast<time_t>(*ptmp - '0');
+      is_last_number  = true;
+    }else if(is_last_number){
+      if('Y' == *ptmp && 1 > last_unit_type){
+        unixtime      += (tmptime * (60 * 60 * 24 * 365));   // average 365 day / year
+        last_unit_type = 1;
+      }else if('M' == *ptmp && 2 > last_unit_type){
+        unixtime      += (tmptime * (60 * 60 * 24 * 30));    // average 30 day / month
+        last_unit_type = 2;
+      }else if('D' == *ptmp && 3 > last_unit_type){
+        unixtime      += (tmptime * (60 * 60 * 24));
+        last_unit_type = 3;
+      }else if('h' == *ptmp && 4 > last_unit_type){
+        unixtime      += (tmptime * (60 * 60));
+        last_unit_type = 4;
+      }else if('m' == *ptmp && 5 > last_unit_type){
+        unixtime      += (tmptime * 60);
+        last_unit_type = 5;
+      }else if('s' == *ptmp && 6 > last_unit_type){
+        unixtime      += tmptime;
+        last_unit_type = 6;
+      }else{
+        return false;
+      }
+      tmptime        = 0;
+      is_last_number = false;
+    }else{
+      return false;
+    }
+  }
+  if(is_last_number){
+    return false;
+  }
+  return true;
+}
+
 std::string s3fs_hex(const unsigned char* input, size_t length)
 {
   std::string hex;
@@ -285,10 +369,10 @@ char* s3fs_base64(const unsigned char* input, size_t length)
   static const char* base = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
   char* result;
 
-  if(!input || 0 >= length){
+  if(!input || 0 == length){
     return NULL;
   }
-  if(NULL == (result = (char*)malloc((((length / 3) + 1) * 4 + 1) * sizeof(char)))){
+  if(NULL == (result = reinterpret_cast<char*>(malloc((((length / 3) + 1) * 4 + 1) * sizeof(char))))){
     return NULL; // ENOMEM
   }
 
@@ -338,7 +422,7 @@ unsigned char* s3fs_decode64(const char* input, size_t* plength)
   if(!input || 0 == strlen(input) || !plength){
     return NULL;
   }
-  if(NULL == (result = (unsigned char*)malloc((strlen(input) + 1)))){
+  if(NULL == (result = reinterpret_cast<unsigned char*>(malloc((strlen(input) + 1))))){
     return NULL; // ENOMEM
   }
 
@@ -367,6 +451,132 @@ unsigned char* s3fs_decode64(const char* input, size_t* plength)
   return result;
 }
 
+/*
+ * detect and rewrite invalid utf8.  We take invalid bytes
+ * and encode them into a private region of the unicode
+ * space.  This is sometimes known as wtf8, wobbly transformation format.
+ * it is necessary because S3 validates the utf8 used for identifiers for
+ * correctness, while some clients may provide invalid utf, notably
+ * windows using cp1252.
+ */
+
+// Base location for transform.  The range 0xE000 - 0xF8ff
+// is a private range, se use the start of this range.
+static unsigned int escape_base = 0xe000;
+
+// encode bytes into wobbly utf8.  
+// 'result' can be null. returns true if transform was needed.
+bool s3fs_wtf8_encode(const char *s, string *result)
+{
+  bool invalid = false;
+
+  // Pass valid utf8 code through
+  for (; *s; s++) {
+    const unsigned char c = *s;
+
+    // single byte encoding
+    if (c <= 0x7f) {
+      if (result)
+	*result += c;
+      continue;
+    }
+
+    // otherwise, it must be one of the valid start bytes
+    if ( c >= 0xc2 && c <= 0xf5 ) {
+
+      // two byte encoding
+      // don't need bounds check, string is zero terminated
+      if ((c & 0xe0) == 0xc0 && (s[1] & 0xc0) == 0x80) {
+        // all two byte encodings starting higher than c1 are valid
+        if (result) {
+          *result += c;
+          *result += *(++s);
+        }
+        continue;
+      } 
+      // three byte encoding
+      if ((c & 0xf0) == 0xe0 && (s[1] & 0xc0) == 0x80 && (s[2] & 0xc0) == 0x80) {
+        const unsigned code = ((c & 0x0f) << 12) | ((s[1] & 0x3f) << 6) | (s[2] & 0x3f);
+        if (code >= 0x800 && ! (code >= 0xd800 && code <= 0xd8ff)) {
+          // not overlong and not a surrogate pair 
+          if (result) {
+            *result += c;
+            *result += *(++s);
+            *result += *(++s);
+          }
+          continue;
+        }
+      }
+      // four byte encoding
+      if ((c & 0xf8) == 0xf0 && (s[1] & 0xc0) == 0x80 && (s[2] & 0xc0) == 0x80 && (s[3] & 0xc0) == 0x80) {
+        const unsigned code = ((c & 0x07) << 18) | ((s[1] & 0x3f) << 12) | ((s[2] & 0x3f) << 6) | (s[3] & 0x3f);
+        if (code >= 0x10000 && code <= 0x10ffff) {
+          // not overlong and in defined unicode space
+          if (result) {
+            *result += c;
+            *result += *(++s);
+            *result += *(++s);
+            *result += *(++s);
+          }
+          continue;
+        }
+      }
+    }
+    // printf("invalid %02x at %d\n", c, i);
+    // Invalid utf8 code.  Convert it to a private two byte area of unicode
+    // e.g. the e000 - f8ff area.  This will be a three byte encoding
+    invalid = true;
+    if (result) {
+      unsigned escape = escape_base + c;
+      *result += 0xe0 | ((escape >> 12) & 0x0f);
+      *result += 0x80 | ((escape >> 06) & 0x3f);
+      *result += 0x80 | ((escape >> 00) & 0x3f);
+    }
+  }
+  return invalid;
+}
+
+string s3fs_wtf8_encode(const string &s)
+{
+  string result;
+  s3fs_wtf8_encode(s.c_str(), &result);
+  return result;
+}
+
+// The reverse operation, turn encoded bytes back into their original values
+// The code assumes that we map to a three-byte code point.
+bool s3fs_wtf8_decode(const char *s, string *result)
+{
+  bool encoded = false;
+  for (; *s; s++) {
+    unsigned char c = *s;
+    // look for a three byte tuple matching our encoding code
+    if ((c & 0xf0) == 0xe0 && (s[1] & 0xc0) == 0x80 && (s[2] & 0xc0) == 0x80) {
+      unsigned code = (c & 0x0f) << 12;
+      code |= (s[1] & 0x3f) << 6;
+      code |= (s[2] & 0x3f) << 0;
+      if (code >= escape_base && code <= escape_base + 0xff) {
+        // convert back
+        encoded = true;
+        if (result)
+          *result += code - escape_base;
+        s+=2;
+        continue;
+      }
+    }
+    if (result)
+      *result += c;
+  }
+  return encoded;
+}
+ 
+string s3fs_wtf8_decode(const string &s)
+{
+  string result;
+  s3fs_wtf8_decode(s.c_str(), &result);
+  return result;
+}
+
 /*
 * Local variables:
 * tab-width: 4

src/string_util.h

@@ -28,17 +28,14 @@
 #include <sys/types.h>
 
 #include <string>
-#include <sstream>
 
-#define SPACES                " \t\r\n"
-#define STR2NCMP(str1, str2)  strncmp(str1, str2, strlen(str2))
+static const std::string SPACES = " \t\r\n";
 
-template<typename T> std::string str(T value) {
-  std::stringstream s;
-  s << value;
-  return s.str();
-}
+static inline int STR2NCMP(const char *str1, const char *str2) { return strncmp(str1, str2, strlen(str2)); }
 
+template <class T> std::string str(T value);
+
+// Convert string to off_t.  Does not signal invalid input.
 off_t s3fs_strtoofft(const char* str, bool is_base_16 = false);
 
 std::string trim_left(const std::string &s, const std::string &t = SPACES);
@@ -49,6 +46,8 @@ std::string get_date_rfc850(void);
 void get_date_sigv3(std::string& date, std::string& date8601);
 std::string get_date_string(time_t tm);
 std::string get_date_iso8601(time_t tm);
+bool get_unixtime_from_iso8601(const char* pdate, time_t& unixtime);
+bool convert_unixtime_from_option_arg(const char* argv, time_t& unixtime);
 std::string urlEncode(const std::string &s);
 std::string urlEncode2(const std::string &s);
 std::string urlDecode(const std::string& s);
@@ -59,6 +58,11 @@ std::string s3fs_hex(const unsigned char* input, size_t length);
 char* s3fs_base64(const unsigned char* input, size_t length);
 unsigned char* s3fs_decode64(const char* input, size_t* plength);
 
+bool s3fs_wtf8_encode(const char *s, std::string *result);
+std::string s3fs_wtf8_encode(const std::string &s);
+bool s3fs_wtf8_decode(const char *s, std::string *result);
+std::string s3fs_wtf8_decode(const std::string &s);
+
 #endif // S3FS_STRING_UTIL_H_
 
 /*

src/test_string_util.cpp

@@ -75,9 +75,47 @@ void test_base64()
   // TODO: invalid input
 }
 
+void test_strtoofft()
+{
+  ASSERT_EQUALS(s3fs_strtoofft("0"), static_cast<off_t>(0L));
+  ASSERT_EQUALS(s3fs_strtoofft("9"), static_cast<off_t>(9L));
+  ASSERT_EQUALS(s3fs_strtoofft("A"), static_cast<off_t>(0L));
+  ASSERT_EQUALS(s3fs_strtoofft("A", /*is_base_16=*/ true), static_cast<off_t>(10L));
+  ASSERT_EQUALS(s3fs_strtoofft("F", /*is_base_16=*/ true), static_cast<off_t>(15L));
+  ASSERT_EQUALS(s3fs_strtoofft("a", /*is_base_16=*/ true), static_cast<off_t>(10L));
+  ASSERT_EQUALS(s3fs_strtoofft("f", /*is_base_16=*/ true), static_cast<off_t>(15L));
+  ASSERT_EQUALS(s3fs_strtoofft("deadbeef", /*is_base_16=*/ true), static_cast<off_t>(3735928559L));
+}
+
+void test_wtf8_encoding()
+{
+  std::string ascii("normal string");
+  std::string utf8("Hyld\xc3\xbdpi \xc3\xbej\xc3\xb3\xc3\xb0""f\xc3\xa9lagsins vex \xc3\xbar k\xc3\xa6rkomnu b\xc3\xb6li \xc3\xad \xc3\xa1st");
+  std::string cp1252("Hyld\xfdpi \xfej\xf3\xf0""f\xe9lagsins vex \xfar k\xe6rkomnu b\xf6li \xed \xe1st");
+  std::string broken = utf8;
+  broken[14] = 0x97;
+  std::string mixed = ascii + utf8 + cp1252;
+
+  ASSERT_EQUALS(s3fs_wtf8_encode(ascii), ascii);
+  ASSERT_EQUALS(s3fs_wtf8_decode(ascii), ascii);
+  ASSERT_EQUALS(s3fs_wtf8_encode(utf8), utf8);
+  ASSERT_EQUALS(s3fs_wtf8_decode(utf8), utf8);
+
+  ASSERT_NEQUALS(s3fs_wtf8_encode(cp1252), cp1252);
+  ASSERT_EQUALS(s3fs_wtf8_decode(s3fs_wtf8_encode(cp1252)), cp1252);
+
+  ASSERT_NEQUALS(s3fs_wtf8_encode(broken), broken);
+  ASSERT_EQUALS(s3fs_wtf8_decode(s3fs_wtf8_encode(broken)), broken);
+
+  ASSERT_NEQUALS(s3fs_wtf8_encode(mixed), mixed);
+  ASSERT_EQUALS(s3fs_wtf8_decode(s3fs_wtf8_encode(mixed)), mixed);
+}
+
 int main(int argc, char *argv[])
 {
   test_trim();
   test_base64();
+  test_strtoofft();
+  test_wtf8_encoding();
   return 0;
 }

src/test_util.h

@@ -20,11 +20,50 @@
 
 #include <cstdlib>
 #include <iostream>
+#include <stdio.h>
 
 template <typename T> void assert_equals(const T &x, const T &y, const char *file, int line)
 {
   if (x != y) {
     std::cerr << x << " != " << y << " at " << file << ":" << line << std::endl;
+    std::cerr << std::endl;
+    std::exit(1);
+  }
+}
+
+template <> void assert_equals(const std::string &x, const std::string &y, const char *file, int line)
+{
+  if (x != y) {
+    std::cerr << x << " != " << y << " at " << file << ":" << line << std::endl;
+    for (unsigned i=0; i<x.length(); i++)
+       fprintf(stderr, "%02x ", (unsigned char)x[i]);
+    std::cerr << std::endl;
+    for (unsigned i=0; i<y.length(); i++)
+       fprintf(stderr, "%02x ", (unsigned char)y[i]);
+    std::cerr << std::endl;
+    std::exit(1);
+  }
+}
+
+
+template <typename T> void assert_nequals(const T &x, const T &y, const char *file, int line)
+{
+  if (x == y) {
+    std::cerr << x << " == " << y << " at " << file << ":" << line << std::endl;
+    std::exit(1);
+  }
+}
+
+template <> void assert_nequals(const std::string &x, const std::string &y, const char *file, int line)
+{
+  if (x == y) {
+    std::cerr << x << " == " << y << " at " << file << ":" << line << std::endl;
+    for (unsigned i=0; i<x.length(); i++)
+       fprintf(stderr, "%02x ", (unsigned char)x[i]);
+    std::cerr << std::endl;
+    for (unsigned i=0; i<y.length(); i++)
+       fprintf(stderr, "%02x ", (unsigned char)y[i]);
+    std::cerr << std::endl;
     std::exit(1);
   }
 }
@@ -34,8 +73,8 @@ void assert_strequals(const char *x, const char *y, const char *file, int line)
   if(x == NULL && y == NULL){
     return;
   // cppcheck-suppress nullPointerRedundantCheck
-  } else if((x == NULL || y == NULL) || strcmp(x, y) != 0){
-    std::cerr << x << " != " << y << " at " << file << ":" << line << std::endl;
+  } else if(x == NULL || y == NULL || strcmp(x, y) != 0){
+    std::cerr << (x ? x : "null") << " != " << (y ? y : "null") << " at " << file << ":" << line << std::endl;
     std::exit(1);
   }
 }
@@ -43,5 +82,8 @@ void assert_strequals(const char *x, const char *y, const char *file, int line)
 #define ASSERT_EQUALS(x, y) \
   assert_equals((x), (y), __FILE__, __LINE__)
 
+#define ASSERT_NEQUALS(x, y) \
+  assert_nequals((x), (y), __FILE__, __LINE__)
+
 #define ASSERT_STREQUALS(x, y) \
   assert_strequals((x), (y), __FILE__, __LINE__)

test/integration-test-common.sh

@@ -41,7 +41,7 @@ set -o errexit
 S3FS=../src/s3fs
 
 # Allow these defaulted values to be overridden
-: ${S3_URL:="http://127.0.0.1:8080"}
+: ${S3_URL:="https://127.0.0.1:8080"}
 : ${S3FS_CREDENTIALS_FILE:="passwd-s3fs"}
 : ${TEST_BUCKET_1:="s3fs-integration-test"}
 
@@ -50,7 +50,7 @@ export S3_URL
 export TEST_SCRIPT_DIR=`pwd`
 export TEST_BUCKET_MOUNT_POINT_1=${TEST_BUCKET_1}
 
-S3PROXY_VERSION="1.5.2"
+S3PROXY_VERSION="1.6.1"
 S3PROXY_BINARY=${S3PROXY_BINARY-"s3proxy-${S3PROXY_VERSION}"}
 
 if [ ! -f "$S3FS_CREDENTIALS_FILE" ]
@@ -108,7 +108,8 @@ function start_s3proxy {
             chmod +x "${S3PROXY_BINARY}"
         fi
 
-        stdbuf -oL -eL java -jar "$S3PROXY_BINARY" --properties $S3PROXY_CONFIG | stdbuf -oL -eL sed -u "s/^/s3proxy: /" &
+        stdbuf -oL -eL java -jar "$S3PROXY_BINARY" --properties $S3PROXY_CONFIG &
+        S3PROXY_PID=$!
 
         # wait for S3Proxy to start
         for i in $(seq 30);
@@ -121,8 +122,6 @@ function start_s3proxy {
             fi
             sleep 1
         done
-
-        S3PROXY_PID=$(netstat -lpnt | grep :8080 | awk '{ print $7 }' | sed -u 's|/java||')
     fi
 }
 
@@ -130,7 +129,6 @@ function stop_s3proxy {
     if [ -n "${S3PROXY_PID}" ]
     then
         kill $S3PROXY_PID
-        wait $S3PROXY_PID
     fi
 }
 
@@ -148,14 +146,14 @@ function start_s3fs {
     # If VALGRIND is set, pass it as options to valgrind.
     # start valgrind-listener in another shell. 
     # eg: VALGRIND="--tool=memcheck --leak-check=full" ./small-integration-test.sh
-    # Start valgind-listener (default port is 1500)
+    # Start valgrind-listener (default port is 1500)
     if [ -n "${VALGRIND}" ]; then
         VALGRIND_EXEC="valgrind ${VALGRIND} --log-socket=127.0.1.1"
     fi
 
     # Common s3fs options:
     #
-    # TODO: Allow all these options to be overriden with env variables
+    # TODO: Allow all these options to be overridden with env variables
     #
     # use_path_request_style
     #     The test env doesn't have virtual hosts
@@ -181,15 +179,35 @@ function start_s3fs {
             $TEST_BUCKET_MOUNT_POINT_1 \
             -o use_path_request_style \
             -o url=${S3_URL} \
+            -o no_check_certificate \
+            -o ssl_verify_hostname=0 \
+            -o use_xattr=1 \
             -o createbucket \
             ${AUTH_OPT} \
             -o dbglevel=${DBGLEVEL:=info} \
+            -o retries=3 \
             -f \
-            ${@} \
-        |& stdbuf -oL -eL sed -u "s/^/s3fs: /" &
+            ${@} | stdbuf -oL -eL sed -u "s/^/s3fs: /" &
     )
 
-    retry 5 grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts || exit 1
+    if [ `uname` = "Darwin" ]; then
+         set +o errexit
+         TRYCOUNT=0
+         while [ $TRYCOUNT -le 20 ]; do
+             df | grep -q $TEST_BUCKET_MOUNT_POINT_1
+             if [ $? -eq 0 ]; then
+                 break;
+             fi
+             sleep 1
+             TRYCOUNT=`expr ${TRYCOUNT} + 1`
+         done
+         if [ $? -ne 0 ]; then
+             exit 1
+         fi
+         set -o errexit
+    else
+        retry 5 grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts || exit 1
+    fi
 
     # Quick way to start system up for manual testing with options under test
     if [[ -n ${INTERACT} ]]; then
@@ -202,14 +220,21 @@ function start_s3fs {
 
 function stop_s3fs {
     # Retry in case file system is in use
-    if grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts; then 
-        retry 10 grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts && fusermount -u $TEST_BUCKET_MOUNT_POINT_1
+    if [ `uname` = "Darwin" ]; then
+        df | grep -q $TEST_BUCKET_MOUNT_POINT_1
+        if [ $? -eq 0 ]; then
+            retry 10 df | grep -q $TEST_BUCKET_MOUNT_POINT_1 && umount $TEST_BUCKET_MOUNT_POINT_1
+        fi
+    else
+        if grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts; then 
+            retry 10 grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts && fusermount -u $TEST_BUCKET_MOUNT_POINT_1
+        fi
     fi
 }
 
 # trap handlers do not stack.  If a test sets its own, the new handler should call common_exit_handler
 function common_exit_handler {
-    stop_s3proxy
     stop_s3fs
+    stop_s3proxy
 }
 trap common_exit_handler EXIT

test/integration-test-main.sh

@@ -6,7 +6,11 @@ source test-utils.sh
 
 function test_append_file {
     describe "Testing append to file ..."
+
     # Write a small test file
+    if [ `uname` = "Darwin" ]; then
+       cat /dev/null > ${TEST_TEXT_FILE}
+    fi
     for x in `seq 1 $TEST_TEXT_FILE_LENGTH`
     do
        echo "echo ${TEST_TEXT} to ${TEST_TEXT_FILE}"
@@ -51,7 +55,11 @@ function test_truncate_empty_file {
     truncate ${TEST_TEXT_FILE} -s $t_size
 
     # Verify file is zero length
-    size=$(stat -c %s ${TEST_TEXT_FILE})
+    if [ `uname` = "Darwin" ]; then
+        size=$(stat -f "%z" ${TEST_TEXT_FILE})
+    else
+        size=$(stat -c %s ${TEST_TEXT_FILE})
+    fi
     if [ $t_size -ne $size ]
     then
         echo "error: expected ${TEST_TEXT_FILE} to be $t_size length, got $size"
@@ -77,6 +85,9 @@ function test_mv_file {
     # create the test file again
     mk_test_file
 
+    # save file length
+    ALT_TEXT_LENGTH=`wc -c $TEST_TEXT_FILE | awk '{print $1}'`
+
     #rename the test file
     mv $TEST_TEXT_FILE $ALT_TEST_TEXT_FILE
     if [ ! -e $ALT_TEST_TEXT_FILE ]
@@ -86,7 +97,6 @@ function test_mv_file {
     fi
 
     # Check the contents of the alt file
-    ALT_TEXT_LENGTH=`echo $TEST_TEXT | wc -c | awk '{print $1}'`
     ALT_FILE_LENGTH=`wc -c $ALT_TEST_TEXT_FILE | awk '{print $1}'`
     if [ "$ALT_FILE_LENGTH" -ne "$ALT_TEXT_LENGTH" ]
     then
@@ -98,7 +108,7 @@ function test_mv_file {
     rm_test_file $ALT_TEST_TEXT_FILE
 }
 
-function test_mv_directory {
+function test_mv_empty_directory {
     describe "Testing mv directory function ..."
     if [ -e $TEST_DIR ]; then
        echo "Unexpected, this file/directory exists: ${TEST_DIR}"
@@ -108,7 +118,6 @@ function test_mv_directory {
     mk_test_dir
 
     mv ${TEST_DIR} ${TEST_DIR}_rename
-
     if [ ! -d "${TEST_DIR}_rename" ]; then
        echo "Directory ${TEST_DIR} was not renamed"
        return 1
@@ -121,6 +130,30 @@ function test_mv_directory {
     fi
 }
 
+function test_mv_nonempty_directory {
+    describe "Testing mv directory function ..."
+    if [ -e $TEST_DIR ]; then
+       echo "Unexpected, this file/directory exists: ${TEST_DIR}"
+       return 1
+    fi
+
+    mk_test_dir
+
+    touch ${TEST_DIR}/file
+
+    mv ${TEST_DIR} ${TEST_DIR}_rename
+    if [ ! -d "${TEST_DIR}_rename" ]; then
+       echo "Directory ${TEST_DIR} was not renamed"
+       return 1
+    fi
+
+    rm -r ${TEST_DIR}_rename
+    if [ -e "${TEST_DIR}_rename" ]; then
+       echo "Could not remove the test directory, it still exists: ${TEST_DIR}_rename"
+       return 1
+    fi
+}
+
 function test_redirects {
     describe "Testing redirects ..."
 
@@ -179,12 +212,21 @@ function test_chmod {
     # create the test file again
     mk_test_file
 
-    ORIGINAL_PERMISSIONS=$(stat --format=%a $TEST_TEXT_FILE)
+    if [ `uname` = "Darwin" ]; then
+        ORIGINAL_PERMISSIONS=$(stat -f "%p" $TEST_TEXT_FILE)
+    else
+        ORIGINAL_PERMISSIONS=$(stat --format=%a $TEST_TEXT_FILE)
+    fi
 
     chmod 777 $TEST_TEXT_FILE;
 
     # if they're the same, we have a problem.
-    if [ $(stat --format=%a $TEST_TEXT_FILE) == $ORIGINAL_PERMISSIONS ]
+    if [ `uname` = "Darwin" ]; then
+        CHANGED_PERMISSIONS=$(stat -f "%p" $TEST_TEXT_FILE)
+    else
+        CHANGED_PERMISSIONS=$(stat --format=%a $TEST_TEXT_FILE)
+    fi
+    if [ $CHANGED_PERMISSIONS == $ORIGINAL_PERMISSIONS ]
     then
       echo "Could not modify $TEST_TEXT_FILE permissions"
       return 1
@@ -200,12 +242,21 @@ function test_chown {
     # create the test file again
     mk_test_file
 
-    ORIGINAL_PERMISSIONS=$(stat --format=%u:%g $TEST_TEXT_FILE)
+    if [ `uname` = "Darwin" ]; then
+        ORIGINAL_PERMISSIONS=$(stat -f "%u:%g" $TEST_TEXT_FILE)
+    else
+        ORIGINAL_PERMISSIONS=$(stat --format=%u:%g $TEST_TEXT_FILE)
+    fi
 
     chown 1000:1000 $TEST_TEXT_FILE;
 
     # if they're the same, we have a problem.
-    if [ $(stat --format=%u:%g $TEST_TEXT_FILE) == $ORIGINAL_PERMISSIONS ]
+    if [ `uname` = "Darwin" ]; then
+        CHANGED_PERMISSIONS=$(stat -f "%u:%g" $TEST_TEXT_FILE)
+    else
+        CHANGED_PERMISSIONS=$(stat --format=%u:%g $TEST_TEXT_FILE)
+    fi
+    if [ $CHANGED_PERMISSIONS == $ORIGINAL_PERMISSIONS ]
     then
       if [ $ORIGINAL_PERMISSIONS == "1000:1000" ]
       then
@@ -262,6 +313,10 @@ function test_rename_before_close {
 
 function test_multipart_upload {
     describe "Testing multi-part upload ..."
+
+    if [ `uname` = "Darwin" ]; then
+       cat /dev/null > $BIG_FILE
+    fi
     dd if=/dev/urandom of="/tmp/${BIG_FILE}" bs=$BIG_FILE_LENGTH count=1
     dd if="/tmp/${BIG_FILE}" of="${BIG_FILE}" bs=$BIG_FILE_LENGTH count=1
 
@@ -278,6 +333,10 @@ function test_multipart_upload {
 
 function test_multipart_copy {
     describe "Testing multi-part copy ..."
+
+    if [ `uname` = "Darwin" ]; then
+       cat /dev/null > $BIG_FILE
+    fi
     dd if=/dev/urandom of="/tmp/${BIG_FILE}" bs=$BIG_FILE_LENGTH count=1
     dd if="/tmp/${BIG_FILE}" of="${BIG_FILE}" bs=$BIG_FILE_LENGTH count=1
     mv "${BIG_FILE}" "${BIG_FILE}-copy"
@@ -364,8 +423,8 @@ function test_mtime_file {
 
     #copy the test file with preserve mode
     cp -p $TEST_TEXT_FILE $ALT_TEST_TEXT_FILE
-    testmtime=`stat -c %Y $TEST_TEXT_FILE`
-    altmtime=`stat -c %Y $ALT_TEST_TEXT_FILE`
+    testmtime=`get_mtime $TEST_TEXT_FILE`
+    altmtime=`get_mtime $ALT_TEST_TEXT_FILE`
     if [ "$testmtime" -ne "$altmtime" ]
     then
        echo "File times do not match:  $testmtime != $altmtime"
@@ -373,6 +432,61 @@ function test_mtime_file {
     fi
 }
 
+function test_update_time() {
+    describe "Testing update time function ..."
+
+    # create the test
+    mk_test_file
+    mtime=`get_ctime $TEST_TEXT_FILE`
+    ctime=`get_mtime $TEST_TEXT_FILE`
+
+    sleep 2
+    chmod +x $TEST_TEXT_FILE
+
+    ctime2=`get_ctime $TEST_TEXT_FILE`
+    mtime2=`get_mtime $TEST_TEXT_FILE`
+    if [ $ctime -eq $ctime2 -o $mtime -ne $mtime2 ]; then
+       echo "Expected updated ctime: $ctime != $ctime2 and same mtime: $mtime == $mtime2"
+       return 1
+    fi
+
+    sleep 2
+    chown $UID:$UID $TEST_TEXT_FILE;
+
+    ctime3=`get_ctime $TEST_TEXT_FILE`
+    mtime3=`get_mtime $TEST_TEXT_FILE`
+    if [ $ctime2 -eq $ctime3 -o $mtime2 -ne $mtime3 ]; then
+       echo "Expected updated ctime: $ctime2 != $ctime3 and same mtime: $mtime2 == $mtime3"
+       return 1
+    fi
+
+    if command -v setfattr >/dev/null 2>&1; then
+        sleep 2
+        setfattr -n key -v value $TEST_TEXT_FILE
+
+        ctime4=`get_ctime $TEST_TEXT_FILE`
+        mtime4=`get_mtime $TEST_TEXT_FILE`
+        if [ $ctime3 -eq $ctime4 -o $mtime3 -ne $mtime4 ]; then
+           echo "Expected updated ctime: $ctime3 != $ctime4 and same mtime: $mtime3 == $mtime4"
+           return 1
+        fi
+    else
+        echo "Skipping extended attribute test"
+        ctime4=`get_ctime $TEST_TEXT_FILE`
+        mtime4=`get_mtime $TEST_TEXT_FILE`
+    fi
+
+    sleep 2
+    echo foo >> $TEST_TEXT_FILE
+
+    ctime5=`get_ctime $TEST_TEXT_FILE`
+    mtime5=`get_mtime $TEST_TEXT_FILE`
+    if [ $ctime4 -eq $ctime5 -o $mtime4 -eq $mtime5 ]; then
+       echo "Expected updated ctime: $ctime4 != $ctime5 and updated mtime: $mtime4 != $mtime5"
+       return 1
+    fi
+}
+
 function test_rm_rf_dir {
    describe "Test that rm -rf will remove directory with contents"
    # Create a dir with some files and directories
@@ -396,29 +510,57 @@ function test_write_after_seek_ahead {
    rm testfile
 }
 
+function test_overwrite_existing_file_range {
+    describe "Test overwrite range succeeds"
+    dd if=<(seq 1000) of=${TEST_TEXT_FILE}
+    dd if=/dev/zero of=${TEST_TEXT_FILE} seek=1 count=1 bs=1024 conv=notrunc
+    cmp ${TEST_TEXT_FILE} <(
+        seq 1000 | head -c 1024
+        dd if=/dev/zero count=1 bs=1024
+        seq 1000 | tail -c +2049
+    )
+    rm -f ${TEST_TEXT_FILE}
+}
+
+function test_concurrency {
+    for i in `seq 10`; do echo foo > $i; done
+    for process in `seq 2`; do
+        for i in `seq 100`; do
+            file=$(ls | sed -n "$(($RANDOM % 10 + 1)){p;q}")
+            cat $file >/dev/null || true
+            rm -f $file
+            echo foo > $i || true
+        done &
+    done
+    wait
+}
+
 
 function add_all_tests {
     add_tests test_append_file 
     add_tests test_truncate_file 
     add_tests test_truncate_empty_file
     add_tests test_mv_file
-    add_tests test_mv_directory
+    add_tests test_mv_empty_directory
+    add_tests test_mv_nonempty_directory
     add_tests test_redirects
     add_tests test_mkdir_rmdir
     add_tests test_chmod
     add_tests test_chown
     add_tests test_list
     add_tests test_remove_nonempty_directory
-    # TODO: broken: https://github.com/s3fs-fuse/s3fs-fuse/issues/145
-    #add_tests test_rename_before_close
+    add_tests test_rename_before_close
     add_tests test_multipart_upload
     add_tests test_multipart_copy
     add_tests test_special_characters
     add_tests test_symlink
     add_tests test_extended_attributes
     add_tests test_mtime_file
+    add_tests test_update_time
     add_tests test_rm_rf_dir
     add_tests test_write_after_seek_ahead
+    add_tests test_overwrite_existing_file_range
+    add_tests test_concurrency
 }
 
 init_suite

test/mergedir.sh

@@ -7,12 +7,12 @@
 ###
 ### UsageFunction <program name>
 ###
-UsageFuntion()
+UsageFunction()
 {
     echo "Usage: $1 [-h] [-y] [-all] <base directory>"
     echo "  -h   print usage"
     echo "  -y   no confirm"
-    echo "  -all force all directoris"
+    echo "  -all force all directories"
     echo "       There is no -all option is only to merge for other S3 client."
     echo "       If -all is specified, this shell script merge all directory"
     echo "       for s3fs old version."
@@ -28,7 +28,7 @@ DIRPARAM=""
 
 while [ "$1" != "" ]; do
     if [ "X$1" = "X-help" -o "X$1" = "X-h" -o "X$1" = "X-H" ]; then
-        UsageFuntion $OWNNAME
+        UsageFunction $OWNNAME
         exit 0
     elif [ "X$1" = "X-y" -o "X$1" = "X-Y" ]; then
         AUTOYES="yes"
@@ -38,7 +38,7 @@ while [ "$1" != "" ]; do
         if [ "X$DIRPARAM" != "X" ]; then
             echo "*** Input error."
             echo ""
-            UsageFuntion $OWNNAME
+            UsageFunction $OWNNAME
             exit 1
         fi
         DIRPARAM=$1
@@ -48,7 +48,7 @@ done
 if [ "X$DIRPARAM" = "X" ]; then
     echo "*** Input error."
     echo ""
-    UsageFuntion $OWNNAME
+    UsageFunction $OWNNAME
     exit 1
 fi
 
@@ -62,7 +62,7 @@ fi
 echo "#############################################################################"
 echo "[CAUTION]"
 echo "This program merges a directory made in s3fs which is older than version 1.64."
-echo "And made in other S3 client appilication."
+echo "And made in other S3 client application."
 echo "This program may be have bugs which are not fixed yet."
 echo "Please execute this program by responsibility of your own."
 echo "#############################################################################"
@@ -104,7 +104,7 @@ for DIR in $DIRLIST; do
     if [ "$ALLYES" = "no" ]; then
         ### Skip "d---------" directories.
         ### Other clients make directory object "dir/" which don't have
-        ### "x-amz-meta-mode" attribyte.
+        ### "x-amz-meta-mode" attribute.
         ### Then these directories is "d---------", it is target directory.
         DIRPERMIT=`ls -ld --time-style=+'%Y%m%d%H%M' $DIR | awk '{print $1}'`
         if [ "$DIRPERMIT" != "d---------" ]; then
@@ -112,7 +112,7 @@ for DIR in $DIRLIST; do
         fi
     fi
 
-    ### Comfirm
+    ### Confirm
     ANSWER=""
     if [ "$AUTOYES" = "yes" ]; then
         ANSWER="y"

test/s3proxy.conf

@@ -1,7 +1,9 @@
-s3proxy.endpoint=http://127.0.0.1:8080
+s3proxy.secure-endpoint=http://127.0.0.1:8080
 s3proxy.authorization=aws-v4
 s3proxy.identity=local-identity
 s3proxy.credential=local-credential
+s3proxy.keystore-path=keystore.jks
+s3proxy.keystore-password=password
 
 jclouds.provider=transient
 jclouds.identity=remote-identity

test/test-utils.sh

@@ -24,6 +24,21 @@ function mk_test_file {
         echo "Could not create file ${TEST_TEXT_FILE}, it does not exist"
         exit 1
     fi
+
+    # wait & check
+    BASE_TEXT_LENGTH=`echo $TEXT | wc -c | awk '{print $1}'`
+    TRY_COUNT=10
+    while true; do
+        MK_TEXT_LENGTH=`wc -c $TEST_TEXT_FILE | awk '{print $1}'`
+        if [ $BASE_TEXT_LENGTH -eq $MK_TEXT_LENGTH ]; then
+            break
+        fi
+        TRY_COUNT=`expr $TRY_COUNT - 1`
+        if [ $TRY_COUNT -le 0 ]; then
+            echo "Could not create file ${TEST_TEXT_FILE}, that file size is something wrong"
+        fi
+        sleep 1
+    done
 }
 
 function rm_test_file {
@@ -65,9 +80,9 @@ function cd_run_dir {
         echo "TEST_BUCKET_MOUNT_POINT variable not set"
         exit 1
     fi
-    RUN_DIR=$(mktemp --directory ${TEST_BUCKET_MOUNT_POINT_1}/testrun-XXXXXX)
+    RUN_DIR=$(mktemp -d ${TEST_BUCKET_MOUNT_POINT_1}/testrun-XXXXXX)
     cd ${RUN_DIR}
-}    
+}
 
 function clean_run_dir {
     if [  -d ${RUN_DIR} ]; then
@@ -154,3 +169,19 @@ function run_suite {
        return 0
    fi
 }
+
+function get_ctime() {
+    if [ `uname` = "Darwin" ]; then
+        stat -f "%c" "$1"
+    else
+        stat -c %Z "$1"
+    fi
+}
+
+function get_mtime() {
+    if [ `uname` = "Darwin" ]; then
+        stat -f "%m" "$1"
+    else
+        stat -c %Y "$1"
+    fi
+}