Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 741831344a | |||
| 7a7c7572ea | |||
| 4c32bc0aa5 | |||
| 0e9cfeb808 | |||
| ae4ae88b6d | |||
| f0c33f8ef2 | |||
| e3a33343b9 | |||
| 20b1c207be | |||
| f1ca5d0340 | |||
| cbec8da9a3 | |||
| 7a55eab399 | |||
| 95f8cab139 | |||
| c1a6d76fc3 | |||
| 08929696f7 | |||
| ba34ba181a | |||
| d2c887a371 | |||
| d5113c0501 | |||
| 29a37645dd | |||
| 601482eff5 | |||
| f141bbd4b4 | |||
| 61020370d5 | |||
| f1f7e76be5 | |||
| 160196798b | |||
| edad91186f | |||
| cd27f0aa54 |
20
Makefile.am
20
Makefile.am
@ -1,3 +1,22 @@
|
||||
######################################################################
|
||||
# s3fs - FUSE-based file system backed by Amazon S3
|
||||
#
|
||||
# Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
######################################################################
|
||||
SUBDIRS=src test doc
|
||||
|
||||
EXTRA_DIST=doc
|
||||
@ -8,3 +27,4 @@ dist-hook:
|
||||
|
||||
release : dist ../utils/release.sh
|
||||
../utils/release.sh $(DIST_ARCHIVES)
|
||||
|
||||
|
||||
2
README
2
README
@ -14,6 +14,8 @@ In order to compile s3fs, You'll need the following requirements:
|
||||
* FUSE (>= 2.8.4)
|
||||
* FUSE Kernel module installed and running (RHEL 4.x/CentOS 4.x users - read below)
|
||||
* OpenSSL-devel (0.9.8)
|
||||
GnuTLS(gcrypt and nettle)
|
||||
NSS
|
||||
* Git
|
||||
|
||||
If you're using YUM or APT to install those packages, then it might require additional packaging, allow it to be installed.
|
||||
|
||||
223
configure.ac
223
configure.ac
@ -26,81 +26,192 @@ AC_CANONICAL_SYSTEM
|
||||
AM_INIT_AUTOMAKE()
|
||||
|
||||
AC_PROG_CXX
|
||||
AC_PROG_CC
|
||||
|
||||
CXXFLAGS="$CXXFLAGS -Wall -D_FILE_OFFSET_BITS=64"
|
||||
|
||||
PKG_CHECK_MODULES([DEPS], [fuse >= 2.8.4 libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9])
|
||||
dnl ----------------------------------------------
|
||||
dnl Choice SSL library
|
||||
dnl ----------------------------------------------
|
||||
auth_lib=na
|
||||
nettle_lib=no
|
||||
|
||||
dnl malloc_trim function
|
||||
AC_CHECK_FUNCS(malloc_trim, , )
|
||||
|
||||
dnl Initializing NSS(temporally)
|
||||
AC_MSG_CHECKING([Initializing libcurl build with NSS])
|
||||
AC_ARG_ENABLE(
|
||||
nss-init,
|
||||
dnl
|
||||
dnl nettle library
|
||||
dnl
|
||||
AC_MSG_CHECKING([s3fs build with nettle(GnuTLS)])
|
||||
AC_ARG_WITH(
|
||||
nettle,
|
||||
[AS_HELP_STRING([--with-nettle], [s3fs build with nettle in GnuTLS(default no)])],
|
||||
[
|
||||
AS_HELP_STRING(
|
||||
[--enable-nss-init],
|
||||
[Inilializing libcurl with NSS (default is no)]
|
||||
)
|
||||
],
|
||||
[
|
||||
case "${enableval}" in
|
||||
case "${withval}" in
|
||||
yes)
|
||||
AC_MSG_RESULT(yes)
|
||||
nss_init_enabled=yes
|
||||
nettle_lib=yes
|
||||
;;
|
||||
*)
|
||||
AC_MSG_RESULT(no)
|
||||
;;
|
||||
esac
|
||||
],
|
||||
[
|
||||
AC_MSG_RESULT(no)
|
||||
])
|
||||
|
||||
dnl
|
||||
dnl use openssl library for ssl
|
||||
dnl
|
||||
AC_MSG_CHECKING([s3fs build with OpenSSL])
|
||||
AC_ARG_WITH(
|
||||
openssl,
|
||||
[AS_HELP_STRING([--with-openssl], [s3fs build with OpenSSL(default is no)])],
|
||||
[
|
||||
case "${withval}" in
|
||||
yes)
|
||||
AC_MSG_RESULT(yes)
|
||||
AS_IF(
|
||||
[test $nettle_lib = no],
|
||||
[auth_lib=openssl],
|
||||
[AC_MSG_ERROR([could not set openssl with nettle, nettle is only for gnutls library])])
|
||||
;;
|
||||
*)
|
||||
AC_MSG_RESULT(no)
|
||||
;;
|
||||
esac
|
||||
],
|
||||
[
|
||||
AC_MSG_RESULT(no)
|
||||
])
|
||||
|
||||
dnl
|
||||
dnl use GnuTLS library for ssl
|
||||
dnl
|
||||
AC_MSG_CHECKING([s3fs build with GnuTLS])
|
||||
AC_ARG_WITH(
|
||||
gnutls,
|
||||
[AS_HELP_STRING([--with-gnutls], [s3fs build with GnuTLS(default is no)])],
|
||||
[
|
||||
case "${withval}" in
|
||||
yes)
|
||||
AC_MSG_RESULT(yes)
|
||||
AS_IF(
|
||||
[test $auth_lib = na],
|
||||
[
|
||||
AS_IF(
|
||||
[test $nettle_lib = no],
|
||||
[auth_lib=gnutls],
|
||||
[auth_lib=nettle])
|
||||
],
|
||||
[AC_MSG_ERROR([could not set gnutls because already set another ssl library])])
|
||||
;;
|
||||
*)
|
||||
AC_MSG_RESULT(no)
|
||||
;;
|
||||
esac
|
||||
],
|
||||
[
|
||||
AC_MSG_RESULT(no)
|
||||
])
|
||||
|
||||
dnl
|
||||
dnl use nss library for ssl
|
||||
dnl
|
||||
AC_MSG_CHECKING([s3fs build with NSS])
|
||||
AC_ARG_WITH(
|
||||
nss,
|
||||
[AS_HELP_STRING([--with-nss], [s3fs build with NSS(default is no)])],
|
||||
[
|
||||
case "${withval}" in
|
||||
yes)
|
||||
AC_MSG_RESULT(yes)
|
||||
AS_IF(
|
||||
[test $auth_lib = na],
|
||||
[
|
||||
AS_IF(
|
||||
[test $nettle_lib = no],
|
||||
[auth_lib=nss],
|
||||
[AC_MSG_ERROR([could not set openssl with nettle, nettle is only for gnutls library])])
|
||||
],
|
||||
[AC_MSG_ERROR([could not set nss because already set another ssl library])])
|
||||
;;
|
||||
*)
|
||||
AC_MSG_RESULT(no)
|
||||
nss_init_enabled=no
|
||||
;;
|
||||
esac
|
||||
],
|
||||
[
|
||||
AC_MSG_RESULT(no)
|
||||
nss_init_enabled=no
|
||||
])
|
||||
|
||||
AS_IF(
|
||||
[test $nss_init_enabled = yes],
|
||||
[
|
||||
AC_DEFINE(NSS_INIT_ENABLED, 1)
|
||||
AC_CHECK_LIB(nss3, NSS_NoDB_Init, , [AC_MSG_ERROR(not found NSS libraries)])
|
||||
AC_CHECK_LIB(plds4, PL_ArenaFinish, , [AC_MSG_ERROR(not found PL_ArenaFinish)])
|
||||
AC_CHECK_LIB(nspr4, PR_Cleanup, , [AC_MSG_ERROR(not found PR_Cleanup)])
|
||||
AC_CHECK_HEADER(nss.h, , [AC_MSG_ERROR(not found nss.h)])
|
||||
AC_CHECK_HEADER(nspr4/prinit.h, , [AC_MSG_ERROR(not found prinit.h)])
|
||||
AC_PATH_PROG(NSSCONFIG, [nss-config], no)
|
||||
AS_IF(
|
||||
[test $NSSCONFIG = no],
|
||||
[
|
||||
DEPS_CFLAGS="$DEPS_CFLAGS -I/usr/include/nss3"
|
||||
DEPS_LIBS="$DEPS_LIBS -lnss3"
|
||||
],
|
||||
[
|
||||
addcflags=`nss-config --cflags`
|
||||
DEPS_CFLAGS="$DEPS_CFLAGS $addcflags"
|
||||
dnl addlib=`nss-config --libs`
|
||||
dnl DEPS_LIBS="$DEPS_LIBS $addlib"
|
||||
DEPS_LIBS="$DEPS_LIBS -lnss3"
|
||||
])
|
||||
AC_PATH_PROG(NSPRCONFIG, [nspr-config], no)
|
||||
AS_IF(
|
||||
[test $NSPRCONFIG = no],
|
||||
[
|
||||
DEPS_CFLAGS="$DEPS_CFLAGS -I/usr/include/nspr4"
|
||||
DEPS_LIBS="$DEPS_LIBS -lnspr4 -lplds4"
|
||||
],
|
||||
[
|
||||
addcflags=`nspr-config --cflags`
|
||||
DEPS_CFLAGS="$DEPS_CFLAGS $addcflags"
|
||||
dnl addlib=`nspr-config --libs`
|
||||
dnl DEPS_LIBS="$DEPS_LIBS $addlib"
|
||||
DEPS_LIBS="$DEPS_LIBS -lnspr4 -lplds4"
|
||||
])
|
||||
])
|
||||
[test $auth_lib = na],
|
||||
AS_IF(
|
||||
[test $nettle_lib = no],
|
||||
[auth_lib=openssl],
|
||||
[AC_MSG_ERROR([could not set nettle without GnuTLS library])]
|
||||
)
|
||||
)
|
||||
|
||||
AS_UNSET(nss_enabled)
|
||||
dnl
|
||||
dnl For PKG_CONFIG before checking nss/gnutls.
|
||||
dnl this is redundant checking, but we need checking before following.
|
||||
dnl
|
||||
PKG_CHECK_MODULES([common_lib_checking], [fuse >= 2.8.4 libcurl >= 7.0 libxml-2.0 >= 2.6])
|
||||
|
||||
AC_MSG_CHECKING([compile s3fs with])
|
||||
case "${auth_lib}" in
|
||||
openssl)
|
||||
AC_MSG_RESULT(OpenSSL)
|
||||
PKG_CHECK_MODULES([DEPS], [fuse >= 2.8.4 libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9])
|
||||
;;
|
||||
gnutls)
|
||||
AC_MSG_RESULT(GnuTLS-gcrypt)
|
||||
gnutls_nettle=""
|
||||
AC_CHECK_LIB(gnutls, gcry_control, [gnutls_nettle=0])
|
||||
AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(gcrypt, gcry_control, [gnutls_nettle=0])])
|
||||
AS_IF([test $gnutls_nettle = 0],
|
||||
[
|
||||
PKG_CHECK_MODULES([DEPS], [fuse >= 2.8.4 libcurl >= 7.0 libxml-2.0 >= 2.6 gnutls >= 2.12.0 ])
|
||||
LIBS="-lgnutls -lgcrypt $LIBS"
|
||||
AC_MSG_CHECKING([gnutls is build with])
|
||||
AC_MSG_RESULT(gcrypt)
|
||||
],
|
||||
[AC_MSG_ERROR([GnuTLS found, but gcrypt not found])])
|
||||
;;
|
||||
nettle)
|
||||
AC_MSG_RESULT(GnuTLS-nettle)
|
||||
gnutls_nettle=""
|
||||
AC_CHECK_LIB(gnutls, nettle_MD5Init, [gnutls_nettle=1])
|
||||
AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(nettle, nettle_MD5Init, [gnutls_nettle=1])])
|
||||
AS_IF([test $gnutls_nettle = 1],
|
||||
[
|
||||
PKG_CHECK_MODULES([DEPS], [fuse >= 2.8.4 libcurl >= 7.0 libxml-2.0 >= 2.6 nettle >= 2.7.1 ])
|
||||
LIBS="-lgnutls -lnettle $LIBS"
|
||||
AC_MSG_CHECKING([gnutls is build with])
|
||||
AC_MSG_RESULT(nettle)
|
||||
],
|
||||
[AC_MSG_ERROR([GnuTLS found, but nettle not found])])
|
||||
;;
|
||||
nss)
|
||||
AC_MSG_RESULT(NSS)
|
||||
PKG_CHECK_MODULES([DEPS], [fuse >= 2.8.4 libcurl >= 7.0 libxml-2.0 >= 2.6 nss >= 3.15.0 ])
|
||||
;;
|
||||
*)
|
||||
AC_MSG_ERROR([unknown ssl library type.])
|
||||
;;
|
||||
esac
|
||||
|
||||
AM_CONDITIONAL([USE_SSL_OPENSSL], [test "$auth_lib" = openssl])
|
||||
AM_CONDITIONAL([USE_SSL_GNUTLS], [test "$auth_lib" = gnutls -o "$auth_lib" = nettle])
|
||||
AM_CONDITIONAL([USE_GNUTLS_NETTLE], [test "$auth_lib" = nettle])
|
||||
AM_CONDITIONAL([USE_SSL_NSS], [test "$auth_lib" = nss])
|
||||
|
||||
dnl ----------------------------------------------
|
||||
dnl end of ssl library
|
||||
dnl ----------------------------------------------
|
||||
|
||||
dnl malloc_trim function
|
||||
AC_CHECK_FUNCS(malloc_trim, , )
|
||||
|
||||
AC_CONFIG_FILES(Makefile src/Makefile test/Makefile doc/Makefile)
|
||||
AC_OUTPUT
|
||||
|
||||
@ -1 +1,21 @@
|
||||
######################################################################
|
||||
# s3fs - FUSE-based file system backed by Amazon S3
|
||||
#
|
||||
# Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
######################################################################
|
||||
dist_man1_MANS = man/s3fs.1
|
||||
|
||||
|
||||
@ -71,9 +71,13 @@ this option can not be specified with use_sse.
|
||||
(can specify use_rrs=1 for old version)
|
||||
.TP
|
||||
\fB\-o\fR use_sse (default is disable)
|
||||
use Amazon's Server Site Encryption.
|
||||
this option can not be specified with use_rrs.
|
||||
(can specify use_sse=1 for old version)
|
||||
use Amazon<EFBFBD>fs Server-Site Encryption or Server-Side Encryption with Customer-Provided Encryption Keys.
|
||||
this option can not be specified with use_rrs. specifying only "use_sse" or "use_sse=1" enables Server-Side Encryption.(use_sse=1 for old version)
|
||||
specifying this option with file path which has some SSE-C secret key enables Server-Side Encryption with Customer-Provided Encryption Keys.(use_sse=file)
|
||||
the file must be 600 permission. the file can have some lines, each line is one SSE-C key. the first line in file is used as Customer-Provided Encryption Keys for uploading and change headers etc.
|
||||
if there are some keys after first line, those are used downloading object which are encripted by not first key.
|
||||
so that, you can keep all SSE-C keys in file, that is SSE-C key history.
|
||||
if AWSSSECKEYS environment is set, you can set SSE-C key instead of this option.
|
||||
.TP
|
||||
\fB\-o\fR passwd_file (default="")
|
||||
specify the path to the password file, which which takes precedence over the password in $HOME/.passwd-s3fs and /etc/passwd-s3fs
|
||||
@ -171,6 +175,9 @@ If you set this option, s3fs do not use PUT with "x-amz-copy-source"(copy api).
|
||||
For a distributed object storage which is compatibility S3 API without PUT(copy api).
|
||||
This option is a subset of nocopyapi option. The nocopyapi option does not use copy-api for all command(ex. chmod, chown, touch, mv, etc), but this option does not use copy-api for only rename command(ex. mv).
|
||||
If this option is specified with nocopapi, the s3fs ignores it.
|
||||
.TP
|
||||
\fB\-o\fR use_path_request_style (use legacy API calling style)
|
||||
Enble compatibility with S3-like APIs which do not support the virtual-host request style, by using the older path request style.
|
||||
.SH FUSE/MOUNT OPTIONS
|
||||
.TP
|
||||
Most of the generic mount options described in 'man mount' are supported (ro, rw, suid, nosuid, dev, nodev, exec, noexec, atime, noatime, sync async, dirsync). Filesystems are mounted with '-onodev,nosuid' by default, which can only be overridden by a privileged user.
|
||||
|
||||
@ -1,7 +1,39 @@
|
||||
######################################################################
|
||||
# s3fs - FUSE-based file system backed by Amazon S3
|
||||
#
|
||||
# Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
######################################################################
|
||||
bin_PROGRAMS=s3fs
|
||||
|
||||
AM_CPPFLAGS = $(DEPS_CFLAGS)
|
||||
if USE_GNUTLS_NETTLE
|
||||
AM_CPPFLAGS += -DUSE_GNUTLS_NETTLE
|
||||
endif
|
||||
|
||||
s3fs_SOURCES = s3fs.cpp s3fs.h curl.cpp curl.h cache.cpp cache.h string_util.cpp string_util.h s3fs_util.cpp s3fs_util.h fdcache.cpp fdcache.h common_auth.cpp s3fs_auth.h common.h
|
||||
if USE_SSL_OPENSSL
|
||||
s3fs_SOURCES += openssl_auth.cpp
|
||||
endif
|
||||
if USE_SSL_GNUTLS
|
||||
s3fs_SOURCES += gnutls_auth.cpp
|
||||
endif
|
||||
if USE_SSL_NSS
|
||||
s3fs_SOURCES += nss_auth.cpp
|
||||
endif
|
||||
|
||||
s3fs_SOURCES = s3fs.cpp s3fs.h curl.cpp curl.h cache.cpp cache.h string_util.cpp string_util.h s3fs_util.cpp s3fs_util.h fdcache.cpp fdcache.h common.h
|
||||
s3fs_LDADD = $(DEPS_LIBS)
|
||||
|
||||
|
||||
@ -440,3 +440,11 @@ bool convert_header_to_stat(const char* path, headers_t& meta, struct stat* pst,
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
|
||||
28
src/cache.h
28
src/cache.h
@ -1,3 +1,22 @@
|
||||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
#ifndef S3FS_CACHE_H_
|
||||
#define S3FS_CACHE_H_
|
||||
|
||||
@ -105,3 +124,12 @@ class StatCache
|
||||
bool convert_header_to_stat(const char* path, headers_t& meta, struct stat* pst, bool forcedir = false);
|
||||
|
||||
#endif // S3FS_CACHE_H_
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
|
||||
29
src/common.h
29
src/common.h
@ -1,3 +1,22 @@
|
||||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
#ifndef S3FS_COMMON_H_
|
||||
#define S3FS_COMMON_H_
|
||||
|
||||
@ -69,6 +88,7 @@ extern bool debug;
|
||||
extern bool foreground;
|
||||
extern bool foreground2;
|
||||
extern bool nomultipart;
|
||||
extern bool pathrequeststyle;
|
||||
extern std::string program_name;
|
||||
extern std::string service_path;
|
||||
extern std::string host;
|
||||
@ -76,3 +96,12 @@ extern std::string bucket;
|
||||
extern std::string mount_prefix;
|
||||
|
||||
#endif // S3FS_COMMON_H_
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
|
||||
112
src/common_auth.cpp
Normal file
112
src/common_auth.cpp
Normal file
@ -0,0 +1,112 @@
|
||||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <string>
|
||||
|
||||
#include "s3fs_auth.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function
|
||||
//-------------------------------------------------------------------
|
||||
char* s3fs_base64(unsigned char* input, size_t length)
|
||||
{
|
||||
static const char* base = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
|
||||
char* result;
|
||||
|
||||
if(!input || 0 >= length){
|
||||
return NULL;
|
||||
}
|
||||
if(NULL == (result = (char*)malloc((((length / 3) + 1) * 4 + 1) * sizeof(char)))){
|
||||
return NULL; // ENOMEM
|
||||
}
|
||||
|
||||
unsigned char parts[4];
|
||||
size_t rpos;
|
||||
size_t wpos;
|
||||
for(rpos = 0, wpos = 0; rpos < length; rpos += 3){
|
||||
parts[0] = (input[rpos] & 0xfc) >> 2;
|
||||
parts[1] = ((input[rpos] & 0x03) << 4) | ((((rpos + 1) < length ? input[rpos + 1] : 0x00) & 0xf0) >> 4);
|
||||
parts[2] = (rpos + 1) < length ? (((input[rpos + 1] & 0x0f) << 2) | ((((rpos + 2) < length ? input[rpos + 2] : 0x00) & 0xc0) >> 6)) : 0x40;
|
||||
parts[3] = (rpos + 2) < length ? (input[rpos + 2] & 0x3f) : 0x40;
|
||||
|
||||
result[wpos++] = base[parts[0]];
|
||||
result[wpos++] = base[parts[1]];
|
||||
result[wpos++] = base[parts[2]];
|
||||
result[wpos++] = base[parts[3]];
|
||||
}
|
||||
result[wpos] = '\0';
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
string s3fs_get_content_md5(int fd)
|
||||
{
|
||||
unsigned char* md5hex;
|
||||
char* base64;
|
||||
string Signature;
|
||||
|
||||
if(NULL == (md5hex = s3fs_md5hexsum(fd, 0, -1))){
|
||||
return string("");
|
||||
}
|
||||
if(NULL == (base64 = s3fs_base64(md5hex, get_md5_digest_length()))){
|
||||
return string(""); // ENOMEM
|
||||
}
|
||||
free(md5hex);
|
||||
|
||||
Signature = base64;
|
||||
free(base64);
|
||||
|
||||
return Signature;
|
||||
}
|
||||
|
||||
string s3fs_md5sum(int fd, off_t start, ssize_t size)
|
||||
{
|
||||
size_t digestlen = get_md5_digest_length();
|
||||
char md5[2 * digestlen + 1];
|
||||
char hexbuf[3];
|
||||
unsigned char* md5hex;
|
||||
|
||||
if(NULL == (md5hex = s3fs_md5hexsum(fd, start, size))){
|
||||
return string("");
|
||||
}
|
||||
|
||||
memset(md5, 0, 2 * digestlen + 1);
|
||||
for(size_t pos = 0; pos < digestlen; pos++){
|
||||
snprintf(hexbuf, 3, "%02x", md5hex[pos]);
|
||||
strncat(md5, hexbuf, 2);
|
||||
}
|
||||
free(md5hex);
|
||||
|
||||
return string(md5);
|
||||
}
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
676
src/curl.cpp
676
src/curl.cpp
@ -31,11 +31,6 @@
|
||||
#include <pthread.h>
|
||||
#include <assert.h>
|
||||
#include <curl/curl.h>
|
||||
#include <openssl/bio.h>
|
||||
#include <openssl/buffer.h>
|
||||
#include <openssl/evp.h>
|
||||
#include <openssl/hmac.h>
|
||||
#include <openssl/md5.h>
|
||||
#include <libxml/xpath.h>
|
||||
#include <libxml/xpathInternals.h>
|
||||
#include <libxml/tree.h>
|
||||
@ -52,9 +47,67 @@
|
||||
#include "string_util.h"
|
||||
#include "s3fs.h"
|
||||
#include "s3fs_util.h"
|
||||
#include "s3fs_auth.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utilities
|
||||
//-------------------------------------------------------------------
|
||||
// [TODO]
|
||||
// This function uses tempolary file, but should not use it.
|
||||
// For not using it, we implement function in each auth file(openssl, nss. gnutls).
|
||||
//
|
||||
static bool make_md5_from_string(const char* pstr, string& md5)
|
||||
{
|
||||
if(!pstr || '\0' == pstr[0]){
|
||||
DPRN("Parameter is wrong.");
|
||||
return false;
|
||||
}
|
||||
FILE* fp;
|
||||
if(NULL == (fp = tmpfile())){
|
||||
FPRN("Could not make tmpfile.");
|
||||
return false;
|
||||
}
|
||||
size_t length = strlen(pstr);
|
||||
if(length != fwrite(pstr, sizeof(char), length, fp)){
|
||||
FPRN("Failed to write tmpfile.");
|
||||
fclose(fp);
|
||||
return false;
|
||||
}
|
||||
int fd;
|
||||
if(0 != fflush(fp) || 0 != fseek(fp, 0L, SEEK_SET) || -1 == (fd = fileno(fp))){
|
||||
FPRN("Failed to make MD5.");
|
||||
fclose(fp);
|
||||
return false;
|
||||
}
|
||||
// base64 md5
|
||||
md5 = s3fs_get_content_md5(fd);
|
||||
if(0 == md5.length()){
|
||||
FPRN("Failed to make MD5.");
|
||||
fclose(fp);
|
||||
return false;
|
||||
}
|
||||
fclose(fp);
|
||||
return true;
|
||||
}
|
||||
|
||||
#if 0 // noused
|
||||
static string tolower_header_name(const char* head)
|
||||
{
|
||||
string::size_type pos;
|
||||
string name = head;
|
||||
string value("");
|
||||
if(string::npos != (pos = name.find(':'))){
|
||||
value= name.substr(pos);
|
||||
name = name.substr(0, pos);
|
||||
}
|
||||
name = lower(name);
|
||||
name += value;
|
||||
return name;
|
||||
}
|
||||
#endif
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Class BodyData
|
||||
//-------------------------------------------------------------------
|
||||
@ -148,7 +201,6 @@ const char* BodyData::str(void) const
|
||||
|
||||
pthread_mutex_t S3fsCurl::curl_handles_lock;
|
||||
pthread_mutex_t S3fsCurl::curl_share_lock[SHARE_MUTEX_MAX];
|
||||
pthread_mutex_t* S3fsCurl::crypt_mutex = NULL;
|
||||
bool S3fsCurl::is_initglobal_done = false;
|
||||
CURLSH* S3fsCurl::hCurlShare = NULL;
|
||||
bool S3fsCurl::is_dns_cache = true; // default
|
||||
@ -159,6 +211,7 @@ int S3fsCurl::retries = 3; // default
|
||||
bool S3fsCurl::is_public_bucket = false;
|
||||
string S3fsCurl::default_acl = "private";
|
||||
bool S3fsCurl::is_use_rrs = false;
|
||||
sseckeylist_t S3fsCurl::sseckeys;
|
||||
bool S3fsCurl::is_use_sse = false;
|
||||
bool S3fsCurl::is_content_md5 = false;
|
||||
bool S3fsCurl::is_verbose = false;
|
||||
@ -168,7 +221,6 @@ string S3fsCurl::AWSAccessToken;
|
||||
time_t S3fsCurl::AWSAccessTokenExpire= 0;
|
||||
string S3fsCurl::IAM_role;
|
||||
long S3fsCurl::ssl_verify_hostname = 1; // default(original code...)
|
||||
const EVP_MD* S3fsCurl::evp_md = EVP_sha1();
|
||||
curltime_t S3fsCurl::curl_times;
|
||||
curlprogress_t S3fsCurl::curl_progress;
|
||||
string S3fsCurl::curl_ca_bundle;
|
||||
@ -340,97 +392,12 @@ void S3fsCurl::UnlockCurlShare(CURL* handle, curl_lock_data nLockData, void* use
|
||||
|
||||
bool S3fsCurl::InitCryptMutex(void)
|
||||
{
|
||||
if(S3fsCurl::crypt_mutex){
|
||||
FPRNNN("crypt_mutex is not NULL, destory it.");
|
||||
if(!S3fsCurl::DestroyCryptMutex()){
|
||||
DPRN("Failed to destroy crypt mutex");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if(NULL == (S3fsCurl::crypt_mutex = static_cast<pthread_mutex_t*>(malloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t))))){
|
||||
DPRNCRIT("Could not allocate memory for crypt mutex");
|
||||
return false;
|
||||
}
|
||||
for(int cnt = 0; cnt < CRYPTO_num_locks(); cnt++){
|
||||
pthread_mutex_init(&S3fsCurl::crypt_mutex[cnt], NULL);
|
||||
}
|
||||
// static lock
|
||||
CRYPTO_set_locking_callback(S3fsCurl::CryptMutexLock);
|
||||
CRYPTO_set_id_callback(S3fsCurl::CryptGetThreadid);
|
||||
// dynamic lock
|
||||
CRYPTO_set_dynlock_create_callback(S3fsCurl::CreateDynCryptMutex);
|
||||
CRYPTO_set_dynlock_lock_callback(S3fsCurl::DynCryptMutexLock);
|
||||
CRYPTO_set_dynlock_destroy_callback(S3fsCurl::DestoryDynCryptMutex);
|
||||
|
||||
return true;
|
||||
return s3fs_init_crypt_mutex();
|
||||
}
|
||||
|
||||
bool S3fsCurl::DestroyCryptMutex(void)
|
||||
{
|
||||
if(!S3fsCurl::crypt_mutex){
|
||||
return true;
|
||||
}
|
||||
|
||||
CRYPTO_set_dynlock_destroy_callback(NULL);
|
||||
CRYPTO_set_dynlock_lock_callback(NULL);
|
||||
CRYPTO_set_dynlock_create_callback(NULL);
|
||||
CRYPTO_set_id_callback(NULL);
|
||||
CRYPTO_set_locking_callback(NULL);
|
||||
|
||||
for(int cnt = 0; cnt < CRYPTO_num_locks(); cnt++){
|
||||
pthread_mutex_destroy(&S3fsCurl::crypt_mutex[cnt]);
|
||||
}
|
||||
CRYPTO_cleanup_all_ex_data();
|
||||
free(S3fsCurl::crypt_mutex);
|
||||
S3fsCurl::crypt_mutex = NULL;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void S3fsCurl::CryptMutexLock(int mode, int pos, const char* file, int line)
|
||||
{
|
||||
if(S3fsCurl::crypt_mutex){
|
||||
if(mode & CRYPTO_LOCK){
|
||||
pthread_mutex_lock(&S3fsCurl::crypt_mutex[pos]);
|
||||
}else{
|
||||
pthread_mutex_unlock(&S3fsCurl::crypt_mutex[pos]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsigned long S3fsCurl::CryptGetThreadid(void)
|
||||
{
|
||||
return (unsigned long)pthread_self();
|
||||
}
|
||||
|
||||
struct CRYPTO_dynlock_value* S3fsCurl::CreateDynCryptMutex(const char* file, int line)
|
||||
{
|
||||
struct CRYPTO_dynlock_value* dyndata;
|
||||
|
||||
if(NULL == (dyndata = (struct CRYPTO_dynlock_value*)malloc(sizeof(struct CRYPTO_dynlock_value)))){
|
||||
return NULL;
|
||||
}
|
||||
pthread_mutex_init(&(dyndata->dyn_mutex), NULL);
|
||||
return dyndata;
|
||||
}
|
||||
|
||||
void S3fsCurl::DynCryptMutexLock(int mode, struct CRYPTO_dynlock_value* dyndata, const char* file, int line)
|
||||
{
|
||||
if(dyndata){
|
||||
if(mode & CRYPTO_LOCK){
|
||||
pthread_mutex_lock(&(dyndata->dyn_mutex));
|
||||
}else{
|
||||
pthread_mutex_unlock(&(dyndata->dyn_mutex));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void S3fsCurl::DestoryDynCryptMutex(struct CRYPTO_dynlock_value* dyndata, const char* file, int line)
|
||||
{
|
||||
if(dyndata){
|
||||
pthread_mutex_destroy(&(dyndata->dyn_mutex));
|
||||
free(dyndata);
|
||||
}
|
||||
return s3fs_destroy_crypt_mutex();
|
||||
}
|
||||
|
||||
// homegrown timeout mechanism
|
||||
@ -775,6 +742,126 @@ bool S3fsCurl::SetUseRrs(bool flag)
|
||||
return old;
|
||||
}
|
||||
|
||||
bool S3fsCurl::PushbackSseKeys(string& onekey)
|
||||
{
|
||||
onekey = trim(onekey);
|
||||
if(0 == onekey.size()){
|
||||
return false;
|
||||
}
|
||||
if('#' == onekey[0]){
|
||||
return false;
|
||||
}
|
||||
// make base64
|
||||
char* pbase64_key;
|
||||
if(NULL == (pbase64_key = s3fs_base64((unsigned char*)onekey.c_str(), onekey.length()))){
|
||||
FPRN("Failed to convert base64 from sse-c key %s", onekey.c_str());
|
||||
return false;
|
||||
}
|
||||
string base64_key = pbase64_key;
|
||||
free(pbase64_key);
|
||||
|
||||
// make MD5
|
||||
string strMd5;
|
||||
if(!make_md5_from_string(onekey.c_str(), strMd5)){
|
||||
FPRN("Could not make MD5 from SSE-C keys(%s).", onekey.c_str());
|
||||
return false;
|
||||
}
|
||||
// mapped MD5 = SSE Key
|
||||
sseckeymap_t md5map;
|
||||
md5map.clear();
|
||||
md5map[strMd5] = base64_key;
|
||||
S3fsCurl::sseckeys.push_back(md5map);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool S3fsCurl::SetSseKeys(const char* filepath)
|
||||
{
|
||||
if(!filepath){
|
||||
DPRN("SSE-C keys filepath is empty.");
|
||||
return false;
|
||||
}
|
||||
S3fsCurl::sseckeys.clear();
|
||||
|
||||
ifstream ssefs(filepath);
|
||||
if(!ssefs.good()){
|
||||
FPRN("Could not open SSE-C keys file(%s).", filepath);
|
||||
return false;
|
||||
}
|
||||
|
||||
string line;
|
||||
while(getline(ssefs, line)){
|
||||
S3fsCurl::PushbackSseKeys(line);
|
||||
}
|
||||
if(0 == S3fsCurl::sseckeys.size()){
|
||||
FPRN("There is no SSE Key in file(%s).", filepath);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool S3fsCurl::LoadEnvSseKeys(void)
|
||||
{
|
||||
char* envkeys = getenv("AWSSSECKEYS");
|
||||
if(NULL == envkeys){
|
||||
return false;
|
||||
}
|
||||
S3fsCurl::sseckeys.clear();
|
||||
|
||||
istringstream fullkeys(envkeys);
|
||||
string onekey;
|
||||
while(getline(fullkeys, onekey, ':')){
|
||||
S3fsCurl::PushbackSseKeys(onekey);
|
||||
}
|
||||
if(0 == S3fsCurl::sseckeys.size()){
|
||||
FPRN("There is no SSE Key in environment(AWSSSECKEYS=%s).", envkeys);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// If md5 is empty, returns first(current) sse key.
|
||||
//
|
||||
bool S3fsCurl::GetSseKey(string& md5, string& ssekey)
|
||||
{
|
||||
for(sseckeylist_t::const_iterator iter = S3fsCurl::sseckeys.begin(); iter != S3fsCurl::sseckeys.end(); iter++){
|
||||
if(0 == md5.length() || md5 == (*iter).begin()->first){
|
||||
md5 = iter->begin()->first;
|
||||
ssekey = iter->begin()->second;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool S3fsCurl::GetSseKeyMd5(int pos, string& md5)
|
||||
{
|
||||
if(pos < 0){
|
||||
return false;
|
||||
}
|
||||
if(S3fsCurl::sseckeys.size() <= static_cast<size_t>(pos)){
|
||||
return false;
|
||||
}
|
||||
int cnt = 0;
|
||||
for(sseckeylist_t::const_iterator iter = S3fsCurl::sseckeys.begin(); iter != S3fsCurl::sseckeys.end(); iter++, cnt++){
|
||||
if(pos == cnt){
|
||||
md5 = iter->begin()->first;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
int S3fsCurl::GetSseKeyCount(void)
|
||||
{
|
||||
return S3fsCurl::sseckeys.size();
|
||||
}
|
||||
|
||||
bool S3fsCurl::IsSseCustomMode(void)
|
||||
{
|
||||
return (0 < S3fsCurl::sseckeys.size());
|
||||
}
|
||||
|
||||
bool S3fsCurl::SetUseSse(bool flag)
|
||||
{
|
||||
bool old = S3fsCurl::is_use_sse;
|
||||
@ -897,7 +984,7 @@ S3fsCurl* S3fsCurl::UploadMultipartPostRetryCallback(S3fsCurl* s3fscurl)
|
||||
return newcurl;
|
||||
}
|
||||
|
||||
int S3fsCurl::ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg)
|
||||
int S3fsCurl::ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd)
|
||||
{
|
||||
int result;
|
||||
string upload_id;
|
||||
@ -923,7 +1010,7 @@ int S3fsCurl::ParallelMultipartUploadRequest(const char* tpath, headers_t& meta,
|
||||
return -errno;
|
||||
}
|
||||
|
||||
if(0 != (result = s3fscurl.PreMultipartPostRequest(tpath, meta, upload_id, ow_sse_flg))){
|
||||
if(0 != (result = s3fscurl.PreMultipartPostRequest(tpath, meta, upload_id, false))){
|
||||
close(fd2);
|
||||
return result;
|
||||
}
|
||||
@ -1002,7 +1089,7 @@ S3fsCurl* S3fsCurl::ParallelGetObjectRetryCallback(S3fsCurl* s3fscurl)
|
||||
// duplicate request(setup new curl object)
|
||||
S3fsCurl* newcurl = new S3fsCurl(s3fscurl->IsUseAhbe());
|
||||
if(0 != (result = newcurl->PreGetObjectRequest(
|
||||
s3fscurl->path.c_str(), s3fscurl->partdata.fd, s3fscurl->partdata.startpos, s3fscurl->partdata.size))){
|
||||
s3fscurl->path.c_str(), s3fscurl->partdata.fd, s3fscurl->partdata.startpos, s3fscurl->partdata.size, s3fscurl->b_ssekey_md5))){
|
||||
DPRN("failed downloading part setup(%d)", result);
|
||||
delete newcurl;
|
||||
return NULL;;
|
||||
@ -1016,6 +1103,12 @@ int S3fsCurl::ParallelGetObjectRequest(const char* tpath, int fd, off_t start, s
|
||||
{
|
||||
FPRNNN("[tpath=%s][fd=%d]", SAFESTRPTR(tpath), fd);
|
||||
|
||||
string sseckeymd5("");
|
||||
char* psseckeymd5;
|
||||
if(NULL != (psseckeymd5 = get_object_sseckey_md5(tpath))){
|
||||
sseckeymd5 = psseckeymd5;
|
||||
free(psseckeymd5);
|
||||
}
|
||||
int result = 0;
|
||||
ssize_t remaining_bytes;
|
||||
|
||||
@ -1036,7 +1129,7 @@ int S3fsCurl::ParallelGetObjectRequest(const char* tpath, int fd, off_t start, s
|
||||
|
||||
// s3fscurl sub object
|
||||
S3fsCurl* s3fscurl_para = new S3fsCurl();
|
||||
if(0 != (result = s3fscurl_para->PreGetObjectRequest(tpath, fd, (start + size - remaining_bytes), chunk))){
|
||||
if(0 != (result = s3fscurl_para->PreGetObjectRequest(tpath, fd, (start + size - remaining_bytes), chunk, sseckeymd5))){
|
||||
DPRN("failed downloading part setup(%d)", result);
|
||||
delete s3fscurl_para;
|
||||
return result;
|
||||
@ -1144,7 +1237,8 @@ bool S3fsCurl::CheckIAMCredentialUpdate(void)
|
||||
S3fsCurl::S3fsCurl(bool ahbe) :
|
||||
hCurl(NULL), path(""), base_path(""), saved_path(""), url(""), requestHeaders(NULL),
|
||||
bodydata(NULL), headdata(NULL), LastResponseCode(-1), postdata(NULL), postdata_remaining(0), is_use_ahbe(ahbe),
|
||||
retry_count(0), b_infile(NULL), b_postdata(NULL), b_postdata_remaining(0), b_partdata_startpos(0), b_partdata_size(0)
|
||||
retry_count(0), b_infile(NULL), b_postdata(NULL), b_postdata_remaining(0), b_partdata_startpos(0), b_partdata_size(0),
|
||||
b_ssekey_pos(-1), b_ssekey_md5("")
|
||||
{
|
||||
type = REQTYPE_UNSET;
|
||||
}
|
||||
@ -1651,10 +1745,6 @@ int S3fsCurl::RequestPerform(void)
|
||||
//
|
||||
string S3fsCurl::CalcSignature(string method, string strMD5, string content_type, string date, string resource)
|
||||
{
|
||||
int ret;
|
||||
int bytes_written;
|
||||
int offset;
|
||||
int write_attempts = 0;
|
||||
string Signature;
|
||||
string StringToSign;
|
||||
|
||||
@ -1682,74 +1772,20 @@ string S3fsCurl::CalcSignature(string method, string strMD5, string content_type
|
||||
int key_len = S3fsCurl::AWSSecretAccessKey.size();
|
||||
const unsigned char* sdata = reinterpret_cast<const unsigned char*>(StringToSign.data());
|
||||
int sdata_len = StringToSign.size();
|
||||
unsigned char md[EVP_MAX_MD_SIZE];
|
||||
unsigned int md_len;
|
||||
unsigned char* md = NULL;
|
||||
unsigned int md_len = 0;;
|
||||
|
||||
HMAC(S3fsCurl::evp_md, key, key_len, sdata, sdata_len, md, &md_len);
|
||||
s3fs_HMAC(key, key_len, sdata, sdata_len, &md, &md_len);
|
||||
|
||||
BIO* b64 = BIO_new(BIO_f_base64());
|
||||
BIO* bmem = BIO_new(BIO_s_mem());
|
||||
b64 = BIO_push(b64, bmem);
|
||||
|
||||
offset = 0;
|
||||
for(;;){
|
||||
bytes_written = BIO_write(b64, &(md[offset]), md_len);
|
||||
write_attempts++;
|
||||
// -1 indicates that an error occurred, or a temporary error, such as
|
||||
// the server is busy, occurred and we need to retry later.
|
||||
// BIO_write can do a short write, this code addresses this condition
|
||||
if(bytes_written <= 0){
|
||||
// Indicates whether a temporary error occurred or a failure to
|
||||
// complete the operation occurred
|
||||
if((ret = BIO_should_retry(b64))){
|
||||
// Wait until the write can be accomplished
|
||||
if(write_attempts <= 10){
|
||||
continue;
|
||||
}
|
||||
// Too many write attempts
|
||||
DPRNNN("Failure during BIO_write, returning null String");
|
||||
CRYPTO_free_ex_data(CRYPTO_EX_INDEX_BIO, b64, &b64->ex_data);
|
||||
BIO_free_all(b64);
|
||||
Signature.clear();
|
||||
return Signature;
|
||||
|
||||
}else{
|
||||
// If not a retry then it is an error
|
||||
DPRNNN("Failure during BIO_write, returning null String");
|
||||
CRYPTO_free_ex_data(CRYPTO_EX_INDEX_BIO, b64, &b64->ex_data);
|
||||
BIO_free_all(b64);
|
||||
Signature.clear();
|
||||
return Signature;
|
||||
}
|
||||
}
|
||||
|
||||
// The write request succeeded in writing some Bytes
|
||||
offset += bytes_written;
|
||||
md_len -= bytes_written;
|
||||
|
||||
// If there is no more data to write, the request sending has been
|
||||
// completed
|
||||
if(md_len <= 0){
|
||||
break;
|
||||
}
|
||||
char* base64;
|
||||
if(NULL == (base64 = s3fs_base64(md, md_len))){
|
||||
free(md);
|
||||
return string(""); // ENOMEM
|
||||
}
|
||||
free(md);
|
||||
|
||||
// Flush the data
|
||||
ret = BIO_flush(b64);
|
||||
if(ret <= 0){
|
||||
DPRNNN("Failure during BIO_flush, returning null String");
|
||||
CRYPTO_free_ex_data(CRYPTO_EX_INDEX_BIO, b64, &b64->ex_data);
|
||||
BIO_free_all(b64);
|
||||
Signature.clear();
|
||||
return Signature;
|
||||
}
|
||||
|
||||
BUF_MEM *bptr;
|
||||
BIO_get_mem_ptr(b64, &bptr);
|
||||
Signature.assign(bptr->data, bptr->length - 1);
|
||||
|
||||
CRYPTO_free_ex_data(CRYPTO_EX_INDEX_BIO, b64, &b64->ex_data);
|
||||
BIO_free_all(b64);
|
||||
Signature = base64;
|
||||
free(base64);
|
||||
|
||||
return Signature;
|
||||
}
|
||||
@ -1877,12 +1913,38 @@ int S3fsCurl::GetIAMCredentials(void)
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// If md5 is empty, build by first(current) sse key
|
||||
//
|
||||
bool S3fsCurl::AddSseKeyRequestHead(string& md5, bool is_copy)
|
||||
{
|
||||
if(!S3fsCurl::IsSseCustomMode()){
|
||||
// Nothing to do
|
||||
return true;
|
||||
}
|
||||
string sseckey;
|
||||
if(S3fsCurl::GetSseKey(md5, sseckey)){
|
||||
if(is_copy){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-algorithm:AES256");
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-copy-source-server-side-encryption-customer-key:" + sseckey).c_str());
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-copy-source-server-side-encryption-customer-key-md5:" + md5).c_str());
|
||||
}else{
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-algorithm:AES256");
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-server-side-encryption-customer-key:" + sseckey).c_str());
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-server-side-encryption-customer-key-md5:" + md5).c_str());
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// tpath : target path for head request
|
||||
// bpath : saved into base_path
|
||||
// savedpath : saved into saved_path
|
||||
// ssekey_pos : -1 means "not use sse", 0 - X means "use sseckey" and "sseckey position".
|
||||
// sseckey position 0 is latest key.
|
||||
//
|
||||
bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char* savedpath)
|
||||
bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char* savedpath, int ssekey_pos)
|
||||
{
|
||||
FPRNINFO("[tpath=%s][bpath=%s][save=%s]", SAFESTRPTR(tpath), SAFESTRPTR(bpath), SAFESTRPTR(savedpath));
|
||||
|
||||
@ -1908,6 +1970,15 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char*
|
||||
string date = get_date();
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str());
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: ");
|
||||
|
||||
if(0 <= ssekey_pos && S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!S3fsCurl::GetSseKeyMd5(ssekey_pos, md5) || !AddSseKeyRequestHead(md5, false)){
|
||||
DPRN("Failed to set SSE-C headers for md5(%s).", md5.c_str());
|
||||
}
|
||||
}
|
||||
b_ssekey_pos = ssekey_pos;
|
||||
|
||||
if(!S3fsCurl::IsPublicBucket()){
|
||||
requestHeaders = curl_slist_sort_insert(
|
||||
requestHeaders,
|
||||
@ -1931,32 +2002,53 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char*
|
||||
|
||||
int S3fsCurl::HeadRequest(const char* tpath, headers_t& meta)
|
||||
{
|
||||
int result;
|
||||
int result = -1;
|
||||
|
||||
FPRNNN("[tpath=%s]", SAFESTRPTR(tpath));
|
||||
|
||||
if(!PreHeadRequest(tpath)){
|
||||
return -1;
|
||||
}
|
||||
// Requests
|
||||
if(0 != (result = RequestPerform())){
|
||||
return result;
|
||||
if(S3fsCurl::IsSseCustomMode()){
|
||||
// SSE-C mode, check all sse-c key at first
|
||||
int pos;
|
||||
for(pos = 0; static_cast<size_t>(pos) < S3fsCurl::sseckeys.size(); pos++){
|
||||
if(0 != pos && !DestroyCurlHandle()){
|
||||
return result;
|
||||
}
|
||||
if(!PreHeadRequest(tpath, NULL, NULL, pos)){
|
||||
return result;
|
||||
}
|
||||
if(0 == (result = RequestPerform())){
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(S3fsCurl::sseckeys.size() <= static_cast<size_t>(pos)){
|
||||
// If sse-c mode is enable, s3fs fails to get head request for normal and sse object.
|
||||
// So try to get head without sse-c header.
|
||||
if(!DestroyCurlHandle() || !PreHeadRequest(tpath, NULL, NULL, -1) || 0 != (result = RequestPerform())){
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}else{
|
||||
// Not sse-c mode
|
||||
if(!PreHeadRequest(tpath) || 0 != (result = RequestPerform())){
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
// file exists in s3
|
||||
// fixme: clean this up.
|
||||
meta.clear();
|
||||
for(headers_t::iterator iter = responseHeaders.begin(); iter != responseHeaders.end(); ++iter){
|
||||
string key = (*iter).first;
|
||||
string value = (*iter).second;
|
||||
if(key == "Content-Type"){
|
||||
if(0 == strcasecmp(key.c_str(), "Content-Type")){
|
||||
meta[key] = value;
|
||||
}else if(key == "Content-Length"){
|
||||
}else if(0 == strcasecmp(key.c_str(), "Content-Length")){
|
||||
meta[key] = value;
|
||||
}else if(key == "ETag"){
|
||||
}else if(0 == strcasecmp(key.c_str(), "ETag")){
|
||||
meta[key] = value;
|
||||
}else if(key == "Last-Modified"){
|
||||
}else if(0 == strcasecmp(key.c_str(), "Last-Modified")){
|
||||
meta[key] = value;
|
||||
}else if(key.substr(0, 5) == "x-amz"){
|
||||
}else if(0 == strcasecmp(key.substr(0, 5).c_str(), "x-amz")){
|
||||
meta[key] = value;
|
||||
}else{
|
||||
// Check for upper case
|
||||
@ -1969,7 +2061,7 @@ int S3fsCurl::HeadRequest(const char* tpath, headers_t& meta)
|
||||
return 0;
|
||||
}
|
||||
|
||||
int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool ow_sse_flg)
|
||||
int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
|
||||
{
|
||||
FPRNNN("[tpath=%s]", SAFESTRPTR(tpath));
|
||||
|
||||
@ -1997,18 +2089,24 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool ow_sse_flg
|
||||
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
|
||||
string key = (*iter).first;
|
||||
string value = (*iter).second;
|
||||
if(key == "Content-Type"){
|
||||
if(0 == strcasecmp(key.c_str(), "Content-Type")){
|
||||
ContentType = value;
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(key.substr(0,9) == "x-amz-acl"){
|
||||
}else if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){
|
||||
// not set value, but after set it.
|
||||
}else if(key.substr(0,10) == "x-amz-meta"){
|
||||
}else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(key == "x-amz-copy-source"){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(!ow_sse_flg && key == "x-amz-server-side-encryption"){
|
||||
// If ow_sse_flg is false, SSE inherit from meta.
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
|
||||
// skip this header, because this header is specified after logic.
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){
|
||||
// skip this header, because this header is specified with "x-amz-...-customer-key-md5".
|
||||
}else if(is_copy && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
// Only copy mode.
|
||||
if(!AddSseKeyRequestHead(value, is_copy)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}
|
||||
}
|
||||
// "x-amz-acl", rrs, sse
|
||||
@ -2016,8 +2114,13 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool ow_sse_flg
|
||||
if(S3fsCurl::is_use_rrs){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY");
|
||||
}
|
||||
if(ow_sse_flg && S3fsCurl::is_use_sse){
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}
|
||||
if(is_use_ahbe){
|
||||
// set additional header by ahbe conf
|
||||
@ -2049,7 +2152,7 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool ow_sse_flg
|
||||
return result;
|
||||
}
|
||||
|
||||
int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg)
|
||||
int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
|
||||
{
|
||||
struct stat st;
|
||||
FILE* file = NULL;
|
||||
@ -2094,7 +2197,7 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse
|
||||
|
||||
string strMD5;
|
||||
if(-1 != fd && S3fsCurl::is_content_md5){
|
||||
strMD5 = GetContentMD5(fd);
|
||||
strMD5 = s3fs_get_content_md5(fd);
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string("Content-MD5: " + strMD5).c_str());
|
||||
}
|
||||
|
||||
@ -2102,16 +2205,19 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse
|
||||
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
|
||||
string key = (*iter).first;
|
||||
string value = (*iter).second;
|
||||
if(key == "Content-Type"){
|
||||
if(0 == strcasecmp(key.c_str(), "Content-Type")){
|
||||
ContentType = value;
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(key.substr(0,9) == "x-amz-acl"){
|
||||
}else if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){
|
||||
// not set value, but after set it.
|
||||
}else if(key.substr(0,10) == "x-amz-meta"){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(!ow_sse_flg && key == "x-amz-server-side-encryption"){
|
||||
// If ow_sse_flg is false, SSE inherit from meta.
|
||||
}else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
|
||||
// skip this header, because this header is specified after logic.
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){
|
||||
// skip this header, because this header is specified after logic.
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
// skip this header, because this header is specified after logic.
|
||||
}
|
||||
}
|
||||
// "x-amz-acl", rrs, sse
|
||||
@ -2119,8 +2225,13 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse
|
||||
if(S3fsCurl::is_use_rrs){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY");
|
||||
}
|
||||
if(ow_sse_flg && S3fsCurl::is_use_sse){
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}
|
||||
if(is_use_ahbe){
|
||||
// set additional header by ahbe conf
|
||||
@ -2160,7 +2271,7 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse
|
||||
return result;
|
||||
}
|
||||
|
||||
int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size)
|
||||
int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size, string& ssekeymd5)
|
||||
{
|
||||
FPRNNN("[tpath=%s][start=%jd][size=%zd]", SAFESTRPTR(tpath), (intmax_t)start, size);
|
||||
|
||||
@ -2190,6 +2301,11 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_
|
||||
range += str(start + size - 1);
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, range.c_str());
|
||||
}
|
||||
if(0 < ssekeymd5.length()){
|
||||
if(!AddSseKeyRequestHead(ssekeymd5, false)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}
|
||||
|
||||
if(!S3fsCurl::IsPublicBucket()){
|
||||
requestHeaders = curl_slist_sort_insert(
|
||||
@ -2212,6 +2328,7 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_
|
||||
partdata.size = size;
|
||||
b_partdata_startpos = start;
|
||||
b_partdata_size = size;
|
||||
b_ssekey_md5 = ssekeymd5;
|
||||
|
||||
type = REQTYPE_GET;
|
||||
|
||||
@ -2227,7 +2344,13 @@ int S3fsCurl::GetObjectRequest(const char* tpath, int fd, off_t start, ssize_t s
|
||||
if(!tpath){
|
||||
return -1;
|
||||
}
|
||||
if(0 != (result = PreGetObjectRequest(tpath, fd, start, size))){
|
||||
string sseckeymd5("");
|
||||
char* psseckeymd5;
|
||||
if(NULL != (psseckeymd5 = get_object_sseckey_md5(tpath))){
|
||||
sseckeymd5 = psseckeymd5;
|
||||
free(psseckeymd5);
|
||||
}
|
||||
if(0 != (result = PreGetObjectRequest(tpath, fd, start, size, sseckeymd5))){
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -2336,7 +2459,7 @@ int S3fsCurl::ListBucketRequest(const char* tpath, const char* query)
|
||||
// Date: Mon, 1 Nov 2010 20:34:56 GMT
|
||||
// Authorization: AWS VGhpcyBtZXNzYWdlIHNpZ25lZCBieSBlbHZpbmc=
|
||||
//
|
||||
int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string& upload_id, bool ow_sse_flg)
|
||||
int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string& upload_id, bool is_copy)
|
||||
{
|
||||
FPRNNN("[tpath=%s]", SAFESTRPTR(tpath));
|
||||
|
||||
@ -2369,13 +2492,19 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
|
||||
string key = (*iter).first;
|
||||
string value = (*iter).second;
|
||||
|
||||
if(key.substr(0,9) == "x-amz-acl"){
|
||||
if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){
|
||||
// not set value, but after set it.
|
||||
}else if(key.substr(0,10) == "x-amz-meta"){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(!ow_sse_flg && key == "x-amz-server-side-encryption"){
|
||||
// If ow_sse_flg is false, SSE inherit from meta.
|
||||
}else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
|
||||
// skip this header, because this header is specified after logic.
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){
|
||||
// skip this header, because this header is specified with "x-amz-...-customer-key-md5".
|
||||
}else if(is_copy && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
// Only copy mode.
|
||||
if(!AddSseKeyRequestHead(value, is_copy)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}
|
||||
}
|
||||
// "x-amz-acl", rrs, sse
|
||||
@ -2383,8 +2512,13 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
|
||||
if(S3fsCurl::is_use_rrs){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY");
|
||||
}
|
||||
if(ow_sse_flg && S3fsCurl::is_use_sse){
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}
|
||||
if(is_use_ahbe){
|
||||
// set additional header by ahbe conf
|
||||
@ -2617,7 +2751,7 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, string&
|
||||
}
|
||||
|
||||
// make md5 and file pointer
|
||||
partdata.etag = md5sum(partdata.fd, partdata.startpos, partdata.size);
|
||||
partdata.etag = s3fs_md5sum(partdata.fd, partdata.startpos, partdata.size);
|
||||
if(partdata.etag.empty()){
|
||||
DPRN("Could not make md5 for file(part %d)", part_num);
|
||||
return -1;
|
||||
@ -2733,12 +2867,12 @@ int S3fsCurl::CopyMultipartPostRequest(const char* from, const char* to, int par
|
||||
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
|
||||
string key = (*iter).first;
|
||||
string value = (*iter).second;
|
||||
if(key == "Content-Type"){
|
||||
if(0 == strcasecmp(key.c_str(), "Content-Type")){
|
||||
ContentType = value;
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(key == "x-amz-copy-source"){
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(key == "x-amz-copy-source-range"){
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source-range")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}
|
||||
// NOTICE: x-amz-acl, x-amz-server-side-encryption is not set!
|
||||
@ -2781,7 +2915,7 @@ int S3fsCurl::CopyMultipartPostRequest(const char* from, const char* to, int par
|
||||
return result;
|
||||
}
|
||||
|
||||
int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta)
|
||||
int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta, bool is_copy)
|
||||
{
|
||||
int result;
|
||||
string upload_id;
|
||||
@ -2792,7 +2926,7 @@ int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& met
|
||||
|
||||
FPRNNN("[tpath=%s]", SAFESTRPTR(tpath));
|
||||
|
||||
if(0 != (result = PreMultipartPostRequest(tpath, meta, upload_id, false))){
|
||||
if(0 != (result = PreMultipartPostRequest(tpath, meta, upload_id, is_copy))){
|
||||
return result;
|
||||
}
|
||||
DestroyCurlHandle();
|
||||
@ -2818,7 +2952,7 @@ int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& met
|
||||
return 0;
|
||||
}
|
||||
|
||||
int S3fsCurl::MultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg)
|
||||
int S3fsCurl::MultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool is_copy)
|
||||
{
|
||||
int result;
|
||||
string upload_id;
|
||||
@ -2844,7 +2978,7 @@ int S3fsCurl::MultipartUploadRequest(const char* tpath, headers_t& meta, int fd,
|
||||
return -errno;
|
||||
}
|
||||
|
||||
if(0 != (result = PreMultipartPostRequest(tpath, meta, upload_id, ow_sse_flg))){
|
||||
if(0 != (result = PreMultipartPostRequest(tpath, meta, upload_id, is_copy))){
|
||||
close(fd2);
|
||||
return result;
|
||||
}
|
||||
@ -2897,7 +3031,7 @@ int S3fsCurl::MultipartRenameRequest(const char* from, const char* to, headers_t
|
||||
meta["Content-Type"] = S3fsCurl::LookupMimeType(string(to));
|
||||
meta["x-amz-copy-source"] = srcresource;
|
||||
|
||||
if(0 != (result = PreMultipartPostRequest(to, meta, upload_id, false))){
|
||||
if(0 != (result = PreMultipartPostRequest(to, meta, upload_id, true))){
|
||||
return result;
|
||||
}
|
||||
DestroyCurlHandle();
|
||||
@ -3407,107 +3541,6 @@ bool AdditionalHeader::Dump(void) const
|
||||
//-------------------------------------------------------------------
|
||||
// Utility functions
|
||||
//-------------------------------------------------------------------
|
||||
string GetContentMD5(int fd)
|
||||
{
|
||||
BIO* b64;
|
||||
BIO* bmem;
|
||||
BUF_MEM* bptr;
|
||||
string Signature;
|
||||
unsigned char* md5hex;
|
||||
|
||||
if(NULL == (md5hex = md5hexsum(fd, 0, -1))){
|
||||
return string("");
|
||||
}
|
||||
|
||||
b64 = BIO_new(BIO_f_base64());
|
||||
bmem = BIO_new(BIO_s_mem());
|
||||
b64 = BIO_push(b64, bmem);
|
||||
|
||||
BIO_write(b64, md5hex, MD5_DIGEST_LENGTH);
|
||||
free(md5hex);
|
||||
if(1 != BIO_flush(b64)){
|
||||
CRYPTO_free_ex_data(CRYPTO_EX_INDEX_BIO, b64, &b64->ex_data);
|
||||
BIO_free_all(b64);
|
||||
return string("");
|
||||
}
|
||||
BIO_get_mem_ptr(b64, &bptr);
|
||||
Signature.assign(bptr->data, bptr->length - 1);
|
||||
|
||||
CRYPTO_free_ex_data(CRYPTO_EX_INDEX_BIO, b64, &b64->ex_data);
|
||||
BIO_free_all(b64);
|
||||
|
||||
return Signature;
|
||||
}
|
||||
|
||||
unsigned char* md5hexsum(int fd, off_t start, ssize_t size)
|
||||
{
|
||||
MD5_CTX c;
|
||||
char buf[512];
|
||||
ssize_t bytes;
|
||||
unsigned char* result;
|
||||
|
||||
if(-1 == size){
|
||||
struct stat st;
|
||||
if(-1 == fstat(fd, &st)){
|
||||
return NULL;
|
||||
}
|
||||
size = static_cast<ssize_t>(st.st_size);
|
||||
}
|
||||
|
||||
// seek to top of file.
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
return NULL;
|
||||
}
|
||||
if(NULL == (result = (unsigned char*)malloc(MD5_DIGEST_LENGTH))){
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memset(buf, 0, 512);
|
||||
MD5_Init(&c);
|
||||
for(ssize_t total = 0; total < size; total += bytes){
|
||||
bytes = 512 < (size - total) ? 512 : (size - total);
|
||||
bytes = read(fd, buf, bytes);
|
||||
if(0 == bytes){
|
||||
// end of file
|
||||
break;
|
||||
}else if(-1 == bytes){
|
||||
// error
|
||||
DPRNNN("file read error(%d)", errno);
|
||||
free(result);
|
||||
return NULL;
|
||||
}
|
||||
MD5_Update(&c, buf, bytes);
|
||||
memset(buf, 0, 512);
|
||||
}
|
||||
MD5_Final(result, &c);
|
||||
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
free(result);
|
||||
return NULL;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
string md5sum(int fd, off_t start, ssize_t size)
|
||||
{
|
||||
char md5[2 * MD5_DIGEST_LENGTH + 1];
|
||||
char hexbuf[3];
|
||||
unsigned char* md5hex;
|
||||
|
||||
if(NULL == (md5hex = md5hexsum(fd, start, size))){
|
||||
return string("");
|
||||
}
|
||||
|
||||
memset(md5, 0, 2 * MD5_DIGEST_LENGTH + 1);
|
||||
for(int i = 0; i < MD5_DIGEST_LENGTH; i++) {
|
||||
snprintf(hexbuf, 3, "%02x", md5hex[i]);
|
||||
strncat(md5, hexbuf, 2);
|
||||
}
|
||||
free(md5hex);
|
||||
|
||||
return string(md5);
|
||||
}
|
||||
|
||||
//
|
||||
// curl_slist_sort_insert
|
||||
// This function is like curl_slist_append function, but this adds data by a-sorting.
|
||||
@ -3591,4 +3624,11 @@ bool MakeUrlResource(const char* realpath, string& resourcepath, string& url)
|
||||
return true;
|
||||
}
|
||||
|
||||
/// END
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
|
||||
75
src/curl.h
75
src/curl.h
@ -1,3 +1,22 @@
|
||||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
#ifndef S3FS_CURL_H_
|
||||
#define S3FS_CURL_H_
|
||||
|
||||
@ -100,18 +119,14 @@ class S3fsMultiCurl;
|
||||
// class S3fsCurl
|
||||
//----------------------------------------------
|
||||
typedef std::map<std::string, std::string> iamcredmap_t;
|
||||
typedef std::map<std::string, std::string> sseckeymap_t;
|
||||
typedef std::list<sseckeymap_t> sseckeylist_t;
|
||||
|
||||
// share
|
||||
#define SHARE_MUTEX_DNS 0
|
||||
#define SHARE_MUTEX_SSL_SESSION 1
|
||||
#define SHARE_MUTEX_MAX 2
|
||||
|
||||
// internal use struct for openssl
|
||||
struct CRYPTO_dynlock_value
|
||||
{
|
||||
pthread_mutex_t dyn_mutex;
|
||||
};
|
||||
|
||||
// Class for lapping curl
|
||||
//
|
||||
class S3fsCurl
|
||||
@ -140,7 +155,6 @@ class S3fsCurl
|
||||
// class variables
|
||||
static pthread_mutex_t curl_handles_lock;
|
||||
static pthread_mutex_t curl_share_lock[SHARE_MUTEX_MAX];
|
||||
static pthread_mutex_t* crypt_mutex;
|
||||
static bool is_initglobal_done;
|
||||
static CURLSH* hCurlShare;
|
||||
static bool is_dns_cache;
|
||||
@ -151,6 +165,7 @@ class S3fsCurl
|
||||
static bool is_public_bucket;
|
||||
static std::string default_acl; // TODO: to enum
|
||||
static bool is_use_rrs;
|
||||
static sseckeylist_t sseckeys;
|
||||
static bool is_use_sse;
|
||||
static bool is_content_md5;
|
||||
static bool is_verbose;
|
||||
@ -160,7 +175,6 @@ class S3fsCurl
|
||||
static time_t AWSAccessTokenExpire;
|
||||
static std::string IAM_role;
|
||||
static long ssl_verify_hostname;
|
||||
static const EVP_MD* evp_md;
|
||||
static curltime_t curl_times;
|
||||
static curlprogress_t curl_progress;
|
||||
static std::string curl_ca_bundle;
|
||||
@ -190,6 +204,8 @@ class S3fsCurl
|
||||
int b_postdata_remaining; // backup for retrying
|
||||
off_t b_partdata_startpos; // backup for retrying
|
||||
ssize_t b_partdata_size; // backup for retrying
|
||||
bool b_ssekey_pos; // backup for retrying
|
||||
std::string b_ssekey_md5; // backup for retrying
|
||||
|
||||
public:
|
||||
// constructor/destructor
|
||||
@ -206,11 +222,6 @@ class S3fsCurl
|
||||
static void UnlockCurlShare(CURL* handle, curl_lock_data nLockData, void* useptr);
|
||||
static bool InitCryptMutex(void);
|
||||
static bool DestroyCryptMutex(void);
|
||||
static void CryptMutexLock(int mode, int pos, const char* file, int line);
|
||||
static unsigned long CryptGetThreadid(void);
|
||||
static struct CRYPTO_dynlock_value* CreateDynCryptMutex(const char* file, int line);
|
||||
static void DynCryptMutexLock(int mode, struct CRYPTO_dynlock_value* dyndata, const char* file, int line);
|
||||
static void DestoryDynCryptMutex(struct CRYPTO_dynlock_value* dyndata, const char* file, int line);
|
||||
static int CurlProgress(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow);
|
||||
|
||||
static bool InitMimeType(const char* MimeFile = NULL);
|
||||
@ -227,6 +238,7 @@ class S3fsCurl
|
||||
|
||||
static bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval);
|
||||
static bool SetIAMCredentials(const char* response);
|
||||
static bool PushbackSseKeys(std::string& onekey);
|
||||
|
||||
// methods
|
||||
bool ResetHandle(void);
|
||||
@ -236,7 +248,7 @@ class S3fsCurl
|
||||
bool GetUploadId(std::string& upload_id);
|
||||
int GetIAMCredentials(void);
|
||||
|
||||
int PreMultipartPostRequest(const char* tpath, headers_t& meta, std::string& upload_id, bool ow_sse_flg);
|
||||
int PreMultipartPostRequest(const char* tpath, headers_t& meta, std::string& upload_id, bool is_copy);
|
||||
int CompleteMultipartPostRequest(const char* tpath, std::string& upload_id, etaglist_t& parts);
|
||||
int UploadMultipartPostSetup(const char* tpath, int part_num, std::string& upload_id);
|
||||
int UploadMultipartPostRequest(const char* tpath, int part_num, std::string& upload_id);
|
||||
@ -246,7 +258,7 @@ class S3fsCurl
|
||||
// class methods
|
||||
static bool InitS3fsCurl(const char* MimeFile = NULL);
|
||||
static bool DestroyS3fsCurl(void);
|
||||
static int ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg);
|
||||
static int ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd);
|
||||
static int ParallelGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size);
|
||||
static bool CheckIAMCredentialUpdate(void);
|
||||
|
||||
@ -263,6 +275,12 @@ class S3fsCurl
|
||||
static std::string SetDefaultAcl(const char* acl);
|
||||
static bool SetUseRrs(bool flag);
|
||||
static bool GetUseRrs(void) { return S3fsCurl::is_use_rrs; }
|
||||
static bool SetSseKeys(const char* filepath);
|
||||
static bool LoadEnvSseKeys(void);
|
||||
static bool GetSseKey(std::string& md5, std::string& ssekey);
|
||||
static bool GetSseKeyMd5(int pos, std::string& md5);
|
||||
static int GetSseKeyCount(void);
|
||||
static bool IsSseCustomMode(void);
|
||||
static bool SetUseSse(bool flag);
|
||||
static bool GetUseSse(void) { return S3fsCurl::is_use_sse; }
|
||||
static bool SetContentMd5(bool flag);
|
||||
@ -285,24 +303,25 @@ class S3fsCurl
|
||||
bool CreateCurlHandle(bool force = false);
|
||||
bool DestroyCurlHandle(void);
|
||||
|
||||
bool AddSseKeyRequestHead(std::string& md5, bool is_copy);
|
||||
bool GetResponseCode(long& responseCode);
|
||||
int RequestPerform(void);
|
||||
int DeleteRequest(const char* tpath);
|
||||
bool PreHeadRequest(const char* tpath, const char* bpath = NULL, const char* savedpath = NULL);
|
||||
bool PreHeadRequest(std::string& tpath, std::string& bpath, std::string& savedpath) {
|
||||
return PreHeadRequest(tpath.c_str(), bpath.c_str(), savedpath.c_str());
|
||||
bool PreHeadRequest(const char* tpath, const char* bpath = NULL, const char* savedpath = NULL, int ssekey_pos = -1);
|
||||
bool PreHeadRequest(std::string& tpath, std::string& bpath, std::string& savedpath, int ssekey_pos = -1) {
|
||||
return PreHeadRequest(tpath.c_str(), bpath.c_str(), savedpath.c_str(), ssekey_pos);
|
||||
}
|
||||
int HeadRequest(const char* tpath, headers_t& meta);
|
||||
int PutHeadRequest(const char* tpath, headers_t& meta, bool ow_sse_flg);
|
||||
int PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg);
|
||||
int PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size);
|
||||
int PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy);
|
||||
int PutRequest(const char* tpath, headers_t& meta, int fd);
|
||||
int PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size, std::string& ssekeymd5);
|
||||
int GetObjectRequest(const char* tpath, int fd, off_t start = -1, ssize_t size = -1);
|
||||
int CheckBucket(void);
|
||||
int ListBucketRequest(const char* tpath, const char* query);
|
||||
int MultipartListRequest(std::string& body);
|
||||
int AbortMultipartUpload(const char* tpath, std::string& upload_id);
|
||||
int MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta);
|
||||
int MultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg);
|
||||
int MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta, bool is_copy);
|
||||
int MultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool is_copy);
|
||||
int MultipartRenameRequest(const char* from, const char* to, headers_t& meta, off_t size);
|
||||
|
||||
// methods(valiables)
|
||||
@ -322,6 +341,7 @@ class S3fsCurl
|
||||
int GetMultipartRetryCount(void) const { return retry_count; }
|
||||
void SetMultipartRetryCount(int retrycnt) { retry_count = retrycnt; }
|
||||
bool IsOverMultipartRetryCount(void) const { return (retry_count >= S3fsCurl::retries); }
|
||||
int GetLastPreHeadSeecKeyPos(void) const { return b_ssekey_pos; }
|
||||
};
|
||||
|
||||
//----------------------------------------------
|
||||
@ -404,3 +424,12 @@ struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* d
|
||||
bool MakeUrlResource(const char* realpath, std::string& resourcepath, std::string& url);
|
||||
|
||||
#endif // S3FS_CURL_H_
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
|
||||
@ -31,7 +31,6 @@
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
#include <curl/curl.h>
|
||||
#include <openssl/crypto.h>
|
||||
#include <string>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
@ -797,7 +796,7 @@ int FdEntity::Load(off_t start, off_t size)
|
||||
break;
|
||||
}
|
||||
// download
|
||||
if((*iter)->bytes >= (2 * S3fsCurl::GetMultipartSize()) && !nomultipart){ // default 20MB
|
||||
if((*iter)->bytes >= static_cast<size_t>(2 * S3fsCurl::GetMultipartSize()) && !nomultipart){ // default 20MB
|
||||
// parallel request
|
||||
// Additional time is needed for large files
|
||||
time_t backup = 0;
|
||||
@ -856,7 +855,7 @@ bool FdEntity::LoadFull(off_t* size, bool force_load)
|
||||
return true;
|
||||
}
|
||||
|
||||
int FdEntity::RowFlush(const char* tpath, headers_t& meta, bool ow_sse_flg, bool force_sync)
|
||||
int FdEntity::RowFlush(const char* tpath, headers_t& meta, bool force_sync)
|
||||
{
|
||||
int result;
|
||||
|
||||
@ -903,13 +902,13 @@ int FdEntity::RowFlush(const char* tpath, headers_t& meta, bool ow_sse_flg, bool
|
||||
if(120 > S3fsCurl::GetReadwriteTimeout()){
|
||||
backup = S3fsCurl::SetReadwriteTimeout(120);
|
||||
}
|
||||
result = S3fsCurl::ParallelMultipartUploadRequest(tpath ? tpath : path.c_str(), meta, fd, ow_sse_flg);
|
||||
result = S3fsCurl::ParallelMultipartUploadRequest(tpath ? tpath : path.c_str(), meta, fd);
|
||||
if(0 != backup){
|
||||
S3fsCurl::SetReadwriteTimeout(backup);
|
||||
}
|
||||
}else{
|
||||
S3fsCurl s3fscurl(true);
|
||||
result = s3fscurl.PutRequest(tpath ? tpath : path.c_str(), meta, fd, ow_sse_flg);
|
||||
result = s3fscurl.PutRequest(tpath ? tpath : path.c_str(), meta, fd);
|
||||
}
|
||||
|
||||
// seek to head of file.
|
||||
@ -1197,3 +1196,11 @@ bool FdManager::Close(FdEntity* ent)
|
||||
return false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
|
||||
@ -1,3 +1,22 @@
|
||||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
#ifndef FD_CACHE_H_
|
||||
#define FD_CACHE_H_
|
||||
|
||||
@ -110,8 +129,8 @@ class FdEntity
|
||||
bool SetAllDisable(void) { return SetAllStatus(false); }
|
||||
bool LoadFull(off_t* size = NULL, bool force_load = false);
|
||||
int Load(off_t start, off_t size);
|
||||
int RowFlush(const char* tpath, headers_t& meta, bool ow_sse_flg, bool force_sync = false);
|
||||
int Flush(headers_t& meta, bool ow_sse_flg, bool force_sync = false) { return RowFlush(NULL, meta, ow_sse_flg, force_sync); }
|
||||
int RowFlush(const char* tpath, headers_t& meta, bool force_sync = false);
|
||||
int Flush(headers_t& meta, bool force_sync = false) { return RowFlush(NULL, meta, force_sync); }
|
||||
ssize_t Read(char* bytes, off_t start, size_t size, bool force_load = false);
|
||||
ssize_t Write(const char* bytes, off_t start, size_t size);
|
||||
};
|
||||
@ -154,3 +173,12 @@ class FdManager
|
||||
};
|
||||
|
||||
#endif // FD_CACHE_H_
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
|
||||
266
src/gnutls_auth.cpp
Normal file
266
src/gnutls_auth.cpp
Normal file
@ -0,0 +1,266 @@
|
||||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
#include <unistd.h>
|
||||
#include <syslog.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <string.h>
|
||||
#include <gcrypt.h>
|
||||
#include <gnutls/gnutls.h>
|
||||
#include <gnutls/crypto.h>
|
||||
#ifdef USE_GNUTLS_NETTLE
|
||||
#include <nettle/md5.h>
|
||||
#include <nettle/sha1.h>
|
||||
#include <nettle/hmac.h>
|
||||
#endif
|
||||
#include <string>
|
||||
#include <map>
|
||||
|
||||
#include "common.h"
|
||||
#include "s3fs_auth.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for version
|
||||
//-------------------------------------------------------------------
|
||||
#ifdef USE_GNUTLS_NETTLE
|
||||
|
||||
const char* s3fs_crypt_lib_name(void)
|
||||
{
|
||||
static const char version[] = "GnuTLS(nettle)";
|
||||
|
||||
return version;
|
||||
}
|
||||
|
||||
#else // USE_GNUTLS_NETTLE
|
||||
|
||||
const char* s3fs_crypt_lib_name(void)
|
||||
{
|
||||
static const char version[] = "GnuTLS(gcrypt)";
|
||||
|
||||
return version;
|
||||
}
|
||||
|
||||
#endif // USE_GNUTLS_NETTLE
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for global init
|
||||
//-------------------------------------------------------------------
|
||||
bool s3fs_init_global_ssl(void)
|
||||
{
|
||||
if(GNUTLS_E_SUCCESS != gnutls_global_init()){
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool s3fs_destroy_global_ssl(void)
|
||||
{
|
||||
gnutls_global_deinit();
|
||||
return true;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for crypt lock
|
||||
//-------------------------------------------------------------------
|
||||
bool s3fs_init_crypt_mutex(void)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
bool s3fs_destroy_crypt_mutex(void)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for HMAC
|
||||
//-------------------------------------------------------------------
|
||||
#ifdef USE_GNUTLS_NETTLE
|
||||
|
||||
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
|
||||
{
|
||||
if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){
|
||||
return false;
|
||||
}
|
||||
|
||||
if(NULL == (*digest = (unsigned char*)malloc(SHA1_DIGEST_SIZE))){
|
||||
return false;
|
||||
}
|
||||
|
||||
struct hmac_sha1_ctx ctx_hmac;
|
||||
hmac_sha1_set_key(&ctx_hmac, keylen, reinterpret_cast<const uint8_t*>(key));
|
||||
hmac_sha1_update(&ctx_hmac, datalen, reinterpret_cast<const uint8_t*>(data));
|
||||
hmac_sha1_digest(&ctx_hmac, SHA1_DIGEST_SIZE, reinterpret_cast<uint8_t*>(*digest));
|
||||
*digestlen = SHA1_DIGEST_SIZE;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
#else // USE_GNUTLS_NETTLE
|
||||
|
||||
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
|
||||
{
|
||||
if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){
|
||||
return false;
|
||||
}
|
||||
|
||||
if(0 >= (*digestlen = gnutls_hmac_get_len(GNUTLS_MAC_SHA1))){
|
||||
return false;
|
||||
}
|
||||
if(NULL == (*digest = (unsigned char*)malloc(*digestlen + 1))){
|
||||
return false;
|
||||
}
|
||||
if(0 > gnutls_hmac_fast(GNUTLS_MAC_SHA1, key, keylen, data, datalen, *digest)){
|
||||
free(*digest);
|
||||
*digest = NULL;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#endif // USE_GNUTLS_NETTLE
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for MD5
|
||||
//-------------------------------------------------------------------
|
||||
#define MD5_DIGEST_LENGTH 16
|
||||
|
||||
size_t get_md5_digest_length(void)
|
||||
{
|
||||
return MD5_DIGEST_LENGTH;
|
||||
}
|
||||
|
||||
#ifdef USE_GNUTLS_NETTLE
|
||||
unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
|
||||
{
|
||||
struct md5_ctx ctx_md5;
|
||||
unsigned char buf[512];
|
||||
ssize_t bytes;
|
||||
unsigned char* result;
|
||||
|
||||
// seek to top of file.
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memset(buf, 0, 512);
|
||||
md5_init(&ctx_md5);
|
||||
|
||||
for(ssize_t total = 0; total < size; total += bytes){
|
||||
bytes = 512 < (size - total) ? 512 : (size - total);
|
||||
bytes = read(fd, buf, bytes);
|
||||
if(0 == bytes){
|
||||
// end of file
|
||||
break;
|
||||
}else if(-1 == bytes){
|
||||
// error
|
||||
DPRNNN("file read error(%d)", errno);
|
||||
return NULL;
|
||||
}
|
||||
md5_update(&ctx_md5, bytes, buf);
|
||||
memset(buf, 0, 512);
|
||||
}
|
||||
if(NULL == (result = (unsigned char*)malloc(get_md5_digest_length()))){
|
||||
return NULL;
|
||||
}
|
||||
md5_digest(&ctx_md5, get_md5_digest_length(), result);
|
||||
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
free(result);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
#else // USE_GNUTLS_NETTLE
|
||||
|
||||
unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
|
||||
{
|
||||
gcry_md_hd_t ctx_md5;
|
||||
gcry_error_t err;
|
||||
char buf[512];
|
||||
ssize_t bytes;
|
||||
unsigned char* result;
|
||||
|
||||
if(-1 == size){
|
||||
struct stat st;
|
||||
if(-1 == fstat(fd, &st)){
|
||||
return NULL;
|
||||
}
|
||||
size = static_cast<ssize_t>(st.st_size);
|
||||
}
|
||||
|
||||
// seek to top of file.
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memset(buf, 0, 512);
|
||||
if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_md5, GCRY_MD_MD5, 0))){
|
||||
DPRNN("MD5 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for(ssize_t total = 0; total < size; total += bytes){
|
||||
bytes = 512 < (size - total) ? 512 : (size - total);
|
||||
bytes = read(fd, buf, bytes);
|
||||
if(0 == bytes){
|
||||
// end of file
|
||||
break;
|
||||
}else if(-1 == bytes){
|
||||
// error
|
||||
DPRNNN("file read error(%d)", errno);
|
||||
return NULL;
|
||||
}
|
||||
gcry_md_write(ctx_md5, buf, bytes);
|
||||
memset(buf, 0, 512);
|
||||
}
|
||||
if(NULL == (result = (unsigned char*)malloc(get_md5_digest_length()))){
|
||||
return NULL;
|
||||
}
|
||||
memcpy(result, gcry_md_read(ctx_md5, 0), get_md5_digest_length());
|
||||
gcry_md_close(ctx_md5);
|
||||
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
free(result);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif // USE_GNUTLS_NETTLE
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
203
src/nss_auth.cpp
Normal file
203
src/nss_auth.cpp
Normal file
@ -0,0 +1,203 @@
|
||||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
#include <unistd.h>
|
||||
#include <syslog.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <string.h>
|
||||
#include <nss.h>
|
||||
#include <pk11pub.h>
|
||||
#include <hasht.h>
|
||||
#include <prinit.h>
|
||||
#include <string>
|
||||
#include <map>
|
||||
|
||||
#include "common.h"
|
||||
#include "s3fs_auth.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for version
|
||||
//-------------------------------------------------------------------
|
||||
const char* s3fs_crypt_lib_name(void)
|
||||
{
|
||||
static const char version[] = "NSS";
|
||||
|
||||
return version;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for global init
|
||||
//-------------------------------------------------------------------
|
||||
bool s3fs_init_global_ssl(void)
|
||||
{
|
||||
NSS_Init(NULL);
|
||||
NSS_NoDB_Init(NULL);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool s3fs_destroy_global_ssl(void)
|
||||
{
|
||||
NSS_Shutdown();
|
||||
PL_ArenaFinish();
|
||||
PR_Cleanup();
|
||||
return true;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for crypt lock
|
||||
//-------------------------------------------------------------------
|
||||
bool s3fs_init_crypt_mutex(void)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
bool s3fs_destroy_crypt_mutex(void)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for HMAC
|
||||
//-------------------------------------------------------------------
|
||||
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
|
||||
{
|
||||
if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){
|
||||
return false;
|
||||
}
|
||||
|
||||
PK11SlotInfo* Slot;
|
||||
PK11SymKey* pKey;
|
||||
PK11Context* Context;
|
||||
SECStatus SecStatus;
|
||||
unsigned char tmpdigest[64];
|
||||
SECItem KeySecItem = {siBuffer, reinterpret_cast<unsigned char*>(const_cast<void*>(key)), keylen};
|
||||
SECItem NullSecItem = {siBuffer, NULL, 0};
|
||||
|
||||
if(NULL == (Slot = PK11_GetInternalKeySlot())){
|
||||
return false;
|
||||
}
|
||||
if(NULL == (pKey = PK11_ImportSymKey(Slot, CKM_SHA_1_HMAC, PK11_OriginUnwrap, CKA_SIGN, &KeySecItem, NULL))){
|
||||
PK11_FreeSlot(Slot);
|
||||
return false;
|
||||
}
|
||||
if(NULL == (Context = PK11_CreateContextBySymKey(CKM_SHA_1_HMAC, CKA_SIGN, pKey, &NullSecItem))){
|
||||
PK11_FreeSymKey(pKey);
|
||||
PK11_FreeSlot(Slot);
|
||||
return false;
|
||||
}
|
||||
|
||||
*digestlen = 0;
|
||||
if(SECSuccess != (SecStatus = PK11_DigestBegin(Context)) ||
|
||||
SECSuccess != (SecStatus = PK11_DigestOp(Context, data, datalen)) ||
|
||||
SECSuccess != (SecStatus = PK11_DigestFinal(Context, tmpdigest, digestlen, sizeof(tmpdigest))) )
|
||||
{
|
||||
PK11_DestroyContext(Context, PR_TRUE);
|
||||
PK11_FreeSymKey(pKey);
|
||||
PK11_FreeSlot(Slot);
|
||||
return false;
|
||||
}
|
||||
PK11_DestroyContext(Context, PR_TRUE);
|
||||
PK11_FreeSymKey(pKey);
|
||||
PK11_FreeSlot(Slot);
|
||||
|
||||
if(NULL == (*digest = (unsigned char*)malloc(*digestlen))){
|
||||
return false;
|
||||
}
|
||||
memcpy(*digest, tmpdigest, *digestlen);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for MD5
|
||||
//-------------------------------------------------------------------
|
||||
size_t get_md5_digest_length(void)
|
||||
{
|
||||
return MD5_LENGTH;
|
||||
}
|
||||
|
||||
unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
|
||||
{
|
||||
PK11Context* md5ctx;
|
||||
unsigned char buf[512];
|
||||
ssize_t bytes;
|
||||
unsigned char* result;
|
||||
unsigned int md5outlen;
|
||||
|
||||
if(-1 == size){
|
||||
struct stat st;
|
||||
if(-1 == fstat(fd, &st)){
|
||||
return NULL;
|
||||
}
|
||||
size = static_cast<ssize_t>(st.st_size);
|
||||
}
|
||||
|
||||
// seek to top of file.
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memset(buf, 0, 512);
|
||||
md5ctx = PK11_CreateDigestContext(SEC_OID_MD5);
|
||||
|
||||
for(ssize_t total = 0; total < size; total += bytes){
|
||||
bytes = 512 < (size - total) ? 512 : (size - total);
|
||||
bytes = read(fd, buf, bytes);
|
||||
if(0 == bytes){
|
||||
// end of file
|
||||
break;
|
||||
}else if(-1 == bytes){
|
||||
// error
|
||||
DPRNNN("file read error(%d)", errno);
|
||||
return NULL;
|
||||
}
|
||||
PK11_DigestOp(md5ctx, buf, bytes);
|
||||
memset(buf, 0, 512);
|
||||
}
|
||||
if(NULL == (result = (unsigned char*)malloc(get_md5_digest_length()))){
|
||||
PK11_DestroyContext(md5ctx, PR_TRUE);
|
||||
return NULL;
|
||||
}
|
||||
PK11_DigestFinal(md5ctx, result, &md5outlen, get_md5_digest_length());
|
||||
PK11_DestroyContext(md5ctx, PR_TRUE);
|
||||
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
free(result);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
263
src/openssl_auth.cpp
Normal file
263
src/openssl_auth.cpp
Normal file
@ -0,0 +1,263 @@
|
||||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
#include <unistd.h>
|
||||
#include <syslog.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <string.h>
|
||||
#include <openssl/bio.h>
|
||||
#include <openssl/buffer.h>
|
||||
#include <openssl/evp.h>
|
||||
#include <openssl/hmac.h>
|
||||
#include <openssl/md5.h>
|
||||
#include <openssl/crypto.h>
|
||||
#include <openssl/err.h>
|
||||
#include <string>
|
||||
#include <map>
|
||||
|
||||
#include "common.h"
|
||||
#include "s3fs_auth.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for version
|
||||
//-------------------------------------------------------------------
|
||||
const char* s3fs_crypt_lib_name(void)
|
||||
{
|
||||
static const char version[] = "OpenSSL";
|
||||
|
||||
return version;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for global init
|
||||
//-------------------------------------------------------------------
|
||||
bool s3fs_init_global_ssl(void)
|
||||
{
|
||||
ERR_load_crypto_strings();
|
||||
ERR_load_BIO_strings();
|
||||
OpenSSL_add_all_algorithms();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool s3fs_destroy_global_ssl(void)
|
||||
{
|
||||
EVP_cleanup();
|
||||
ERR_free_strings();
|
||||
return true;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for crypt lock
|
||||
//-------------------------------------------------------------------
|
||||
// internal use struct for openssl
|
||||
struct CRYPTO_dynlock_value
|
||||
{
|
||||
pthread_mutex_t dyn_mutex;
|
||||
};
|
||||
|
||||
static pthread_mutex_t* s3fs_crypt_mutex = NULL;
|
||||
|
||||
static void s3fs_crypt_mutex_lock(int mode, int pos, const char* file, int line)
|
||||
{
|
||||
if(s3fs_crypt_mutex){
|
||||
if(mode & CRYPTO_LOCK){
|
||||
pthread_mutex_lock(&s3fs_crypt_mutex[pos]);
|
||||
}else{
|
||||
pthread_mutex_unlock(&s3fs_crypt_mutex[pos]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static unsigned long s3fs_crypt_get_threadid(void)
|
||||
{
|
||||
return static_cast<unsigned long>(pthread_self());
|
||||
}
|
||||
|
||||
static struct CRYPTO_dynlock_value* s3fs_dyn_crypt_mutex(const char* file, int line)
|
||||
{
|
||||
struct CRYPTO_dynlock_value* dyndata;
|
||||
|
||||
if(NULL == (dyndata = static_cast<struct CRYPTO_dynlock_value*>(malloc(sizeof(struct CRYPTO_dynlock_value))))){
|
||||
DPRNCRIT("Could not allocate memory for CRYPTO_dynlock_value");
|
||||
return NULL;
|
||||
}
|
||||
pthread_mutex_init(&(dyndata->dyn_mutex), NULL);
|
||||
return dyndata;
|
||||
}
|
||||
|
||||
static void s3fs_dyn_crypt_mutex_lock(int mode, struct CRYPTO_dynlock_value* dyndata, const char* file, int line)
|
||||
{
|
||||
if(dyndata){
|
||||
if(mode & CRYPTO_LOCK){
|
||||
pthread_mutex_lock(&(dyndata->dyn_mutex));
|
||||
}else{
|
||||
pthread_mutex_unlock(&(dyndata->dyn_mutex));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void s3fs_destroy_dyn_crypt_mutex(struct CRYPTO_dynlock_value* dyndata, const char* file, int line)
|
||||
{
|
||||
if(dyndata){
|
||||
pthread_mutex_destroy(&(dyndata->dyn_mutex));
|
||||
free(dyndata);
|
||||
}
|
||||
}
|
||||
|
||||
bool s3fs_init_crypt_mutex(void)
|
||||
{
|
||||
if(s3fs_crypt_mutex){
|
||||
FPRNNN("s3fs_crypt_mutex is not NULL, destory it.");
|
||||
if(!s3fs_destroy_crypt_mutex()){
|
||||
DPRN("Failed to s3fs_crypt_mutex");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if(NULL == (s3fs_crypt_mutex = static_cast<pthread_mutex_t*>(malloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t))))){
|
||||
DPRNCRIT("Could not allocate memory for s3fs_crypt_mutex");
|
||||
return false;
|
||||
}
|
||||
for(int cnt = 0; cnt < CRYPTO_num_locks(); cnt++){
|
||||
pthread_mutex_init(&s3fs_crypt_mutex[cnt], NULL);
|
||||
}
|
||||
// static lock
|
||||
CRYPTO_set_locking_callback(s3fs_crypt_mutex_lock);
|
||||
CRYPTO_set_id_callback(s3fs_crypt_get_threadid);
|
||||
// dynamic lock
|
||||
CRYPTO_set_dynlock_create_callback(s3fs_dyn_crypt_mutex);
|
||||
CRYPTO_set_dynlock_lock_callback(s3fs_dyn_crypt_mutex_lock);
|
||||
CRYPTO_set_dynlock_destroy_callback(s3fs_destroy_dyn_crypt_mutex);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool s3fs_destroy_crypt_mutex(void)
|
||||
{
|
||||
if(!s3fs_crypt_mutex){
|
||||
return true;
|
||||
}
|
||||
|
||||
CRYPTO_set_dynlock_destroy_callback(NULL);
|
||||
CRYPTO_set_dynlock_lock_callback(NULL);
|
||||
CRYPTO_set_dynlock_create_callback(NULL);
|
||||
CRYPTO_set_id_callback(NULL);
|
||||
CRYPTO_set_locking_callback(NULL);
|
||||
|
||||
for(int cnt = 0; cnt < CRYPTO_num_locks(); cnt++){
|
||||
pthread_mutex_destroy(&s3fs_crypt_mutex[cnt]);
|
||||
}
|
||||
CRYPTO_cleanup_all_ex_data();
|
||||
free(s3fs_crypt_mutex);
|
||||
s3fs_crypt_mutex = NULL;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for HMAC
|
||||
//-------------------------------------------------------------------
|
||||
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
|
||||
{
|
||||
if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){
|
||||
return false;
|
||||
}
|
||||
(*digestlen) = EVP_MAX_MD_SIZE * sizeof(unsigned char);
|
||||
if(NULL == ((*digest) = (unsigned char*)malloc(*digestlen))){
|
||||
return false;
|
||||
}
|
||||
HMAC(EVP_sha1(), key, keylen, data, datalen, *digest, digestlen);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for MD5
|
||||
//-------------------------------------------------------------------
|
||||
size_t get_md5_digest_length(void)
|
||||
{
|
||||
return MD5_DIGEST_LENGTH;
|
||||
}
|
||||
|
||||
unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
|
||||
{
|
||||
MD5_CTX md5ctx;
|
||||
char buf[512];
|
||||
ssize_t bytes;
|
||||
unsigned char* result;
|
||||
|
||||
if(-1 == size){
|
||||
struct stat st;
|
||||
if(-1 == fstat(fd, &st)){
|
||||
return NULL;
|
||||
}
|
||||
size = static_cast<ssize_t>(st.st_size);
|
||||
}
|
||||
|
||||
// seek to top of file.
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memset(buf, 0, 512);
|
||||
MD5_Init(&md5ctx);
|
||||
|
||||
for(ssize_t total = 0; total < size; total += bytes){
|
||||
bytes = 512 < (size - total) ? 512 : (size - total);
|
||||
bytes = read(fd, buf, bytes);
|
||||
if(0 == bytes){
|
||||
// end of file
|
||||
break;
|
||||
}else if(-1 == bytes){
|
||||
// error
|
||||
DPRNNN("file read error(%d)", errno);
|
||||
return NULL;
|
||||
}
|
||||
MD5_Update(&md5ctx, buf, bytes);
|
||||
memset(buf, 0, 512);
|
||||
}
|
||||
|
||||
if(NULL == (result = (unsigned char*)malloc(get_md5_digest_length()))){
|
||||
return NULL;
|
||||
}
|
||||
MD5_Final(result, &md5ctx);
|
||||
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
free(result);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
207
src/s3fs.cpp
207
src/s3fs.cpp
@ -31,7 +31,6 @@
|
||||
#include <libxml/xpathInternals.h>
|
||||
#include <libxml/tree.h>
|
||||
#include <curl/curl.h>
|
||||
#include <openssl/crypto.h>
|
||||
#include <pwd.h>
|
||||
#include <grp.h>
|
||||
#include <getopt.h>
|
||||
@ -50,6 +49,7 @@
|
||||
#include "string_util.h"
|
||||
#include "s3fs_util.h"
|
||||
#include "fdcache.h"
|
||||
#include "s3fs_auth.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@ -83,6 +83,7 @@ bool debug = false;
|
||||
bool foreground = false;
|
||||
bool foreground2 = false;
|
||||
bool nomultipart = false;
|
||||
bool pathrequeststyle = false;
|
||||
std::string program_name;
|
||||
std::string service_path = "/";
|
||||
std::string host = "http://s3.amazonaws.com";
|
||||
@ -134,7 +135,7 @@ static xmlChar* get_base_exp(xmlDocPtr doc, const char* exp);
|
||||
static xmlChar* get_prefix(xmlDocPtr doc);
|
||||
static xmlChar* get_next_marker(xmlDocPtr doc);
|
||||
static char* get_object_name(xmlDocPtr doc, xmlNodePtr node, const char* path);
|
||||
static int put_headers(const char* path, headers_t& meta, bool ow_sse_flg);
|
||||
static int put_headers(const char* path, headers_t& meta, bool is_copy);
|
||||
static int rename_large_object(const char* from, const char* to);
|
||||
static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gid);
|
||||
static int create_directory_object(const char* path, mode_t mode, time_t time, uid_t uid, gid_t gid);
|
||||
@ -609,6 +610,30 @@ static int check_parent_object_access(const char* path, int mask)
|
||||
return 0;
|
||||
}
|
||||
|
||||
//
|
||||
// This function is global, is called fom curl class(GetObject).
|
||||
//
|
||||
char* get_object_sseckey_md5(const char* path)
|
||||
{
|
||||
if(!path){
|
||||
return NULL;
|
||||
}
|
||||
headers_t meta;
|
||||
|
||||
if(0 != get_object_attribute(path, NULL, &meta)){
|
||||
DPRNNN("Failed to get object(%s) headers", path);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
|
||||
string key = (*iter).first;
|
||||
if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
return strdup((*iter).second.c_str());
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static FdEntity* get_local_fent(const char* path, bool is_load)
|
||||
{
|
||||
struct stat stobj;
|
||||
@ -643,7 +668,7 @@ static FdEntity* get_local_fent(const char* path, bool is_load)
|
||||
* ow_sse_flg is for over writing sse header by use_sse option.
|
||||
* @return fuse return code
|
||||
*/
|
||||
static int put_headers(const char* path, headers_t& meta, bool ow_sse_flg)
|
||||
static int put_headers(const char* path, headers_t& meta, bool is_copy)
|
||||
{
|
||||
int result;
|
||||
S3fsCurl s3fscurl(true);
|
||||
@ -658,11 +683,11 @@ static int put_headers(const char* path, headers_t& meta, bool ow_sse_flg)
|
||||
|
||||
if(buf.st_size >= FIVE_GB){
|
||||
// multipart
|
||||
if(0 != (result = s3fscurl.MultipartHeadRequest(path, buf.st_size, meta))){
|
||||
if(0 != (result = s3fscurl.MultipartHeadRequest(path, buf.st_size, meta, is_copy))){
|
||||
return result;
|
||||
}
|
||||
}else{
|
||||
if(0 != (result = s3fscurl.PutHeadRequest(path, meta, ow_sse_flg))){
|
||||
if(0 != (result = s3fscurl.PutHeadRequest(path, meta, is_copy))){
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@ -766,7 +791,7 @@ static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gi
|
||||
meta["x-amz-meta-mtime"] = str(time(NULL));
|
||||
|
||||
S3fsCurl s3fscurl(true);
|
||||
return s3fscurl.PutRequest(path, meta, -1, false); // fd=-1 means for creating zero byte object.
|
||||
return s3fscurl.PutRequest(path, meta, -1); // fd=-1 means for creating zero byte object.
|
||||
}
|
||||
|
||||
static int s3fs_mknod(const char *path, mode_t mode, dev_t rdev)
|
||||
@ -851,7 +876,7 @@ static int create_directory_object(const char* path, mode_t mode, time_t time, u
|
||||
meta["x-amz-meta-mtime"] = str(time);
|
||||
|
||||
S3fsCurl s3fscurl;
|
||||
return s3fscurl.PutRequest(tpath.c_str(), meta, -1, false); // fd=-1 means for creating zero byte object.
|
||||
return s3fscurl.PutRequest(tpath.c_str(), meta, -1); // fd=-1 means for creating zero byte object.
|
||||
}
|
||||
|
||||
static int s3fs_mkdir(const char* path, mode_t mode)
|
||||
@ -1013,7 +1038,7 @@ static int s3fs_symlink(const char* from, const char* to)
|
||||
return -errno;
|
||||
}
|
||||
// upload
|
||||
if(0 != (result = ent->Flush(headers, true, true))){
|
||||
if(0 != (result = ent->Flush(headers, true))){
|
||||
DPRN("could not upload tmpfile(result=%d)", result);
|
||||
}
|
||||
FdManager::get()->Close(ent);
|
||||
@ -1049,7 +1074,7 @@ static int rename_object(const char* from, const char* to)
|
||||
meta["Content-Type"] = S3fsCurl::LookupMimeType(string(to));
|
||||
meta["x-amz-metadata-directive"] = "REPLACE";
|
||||
|
||||
if(0 != (result = put_headers(to, meta, false))){
|
||||
if(0 != (result = put_headers(to, meta, true))){
|
||||
return result;
|
||||
}
|
||||
result = s3fs_unlink(from);
|
||||
@ -1090,7 +1115,7 @@ static int rename_object_nocopy(const char* from, const char* to)
|
||||
}
|
||||
|
||||
// upload
|
||||
if(0 != (result = ent->RowFlush(to, meta, false, true))){
|
||||
if(0 != (result = ent->RowFlush(to, meta, true))){
|
||||
DPRN("could not upload file(%s): result=%d", to, result);
|
||||
FdManager::get()->Close(ent);
|
||||
return result;
|
||||
@ -1390,7 +1415,7 @@ static int s3fs_chmod(const char* path, mode_t mode)
|
||||
meta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
|
||||
meta["x-amz-metadata-directive"] = "REPLACE";
|
||||
|
||||
if(put_headers(strpath.c_str(), meta, false) != 0){
|
||||
if(put_headers(strpath.c_str(), meta, true) != 0){
|
||||
return -EIO;
|
||||
}
|
||||
StatCache::getStatCacheData()->DelStat(nowcache);
|
||||
@ -1466,7 +1491,7 @@ static int s3fs_chmod_nocopy(const char* path, mode_t mode)
|
||||
}
|
||||
|
||||
// upload
|
||||
if(0 != (result = ent->Flush(meta, false, true))){
|
||||
if(0 != (result = ent->Flush(meta, true))){
|
||||
DPRN("could not upload file(%s): result=%d", strpath.c_str(), result);
|
||||
FdManager::get()->Close(ent);
|
||||
return result;
|
||||
@ -1552,7 +1577,7 @@ static int s3fs_chown(const char* path, uid_t uid, gid_t gid)
|
||||
meta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
|
||||
meta["x-amz-metadata-directive"] = "REPLACE";
|
||||
|
||||
if(put_headers(strpath.c_str(), meta, false) != 0){
|
||||
if(put_headers(strpath.c_str(), meta, true) != 0){
|
||||
return -EIO;
|
||||
}
|
||||
StatCache::getStatCacheData()->DelStat(nowcache);
|
||||
@ -1638,7 +1663,7 @@ static int s3fs_chown_nocopy(const char* path, uid_t uid, gid_t gid)
|
||||
}
|
||||
|
||||
// upload
|
||||
if(0 != (result = ent->Flush(meta, false, true))){
|
||||
if(0 != (result = ent->Flush(meta, true))){
|
||||
DPRN("could not upload file(%s): result=%d", strpath.c_str(), result);
|
||||
FdManager::get()->Close(ent);
|
||||
return result;
|
||||
@ -1710,7 +1735,7 @@ static int s3fs_utimens(const char* path, const struct timespec ts[2])
|
||||
meta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
|
||||
meta["x-amz-metadata-directive"] = "REPLACE";
|
||||
|
||||
if(put_headers(strpath.c_str(), meta, false) != 0){
|
||||
if(put_headers(strpath.c_str(), meta, true) != 0){
|
||||
return -EIO;
|
||||
}
|
||||
StatCache::getStatCacheData()->DelStat(nowcache);
|
||||
@ -1795,7 +1820,7 @@ static int s3fs_utimens_nocopy(const char* path, const struct timespec ts[2])
|
||||
}
|
||||
|
||||
// upload
|
||||
if(0 != (result = ent->Flush(meta, false, true))){
|
||||
if(0 != (result = ent->Flush(meta, true))){
|
||||
DPRN("could not upload file(%s): result=%d", strpath.c_str(), result);
|
||||
FdManager::get()->Close(ent);
|
||||
return result;
|
||||
@ -1846,7 +1871,7 @@ static int s3fs_truncate(const char* path, off_t size)
|
||||
}
|
||||
|
||||
// upload
|
||||
if(0 != (result = ent->Flush(meta, false, true))){
|
||||
if(0 != (result = ent->Flush(meta, true))){
|
||||
DPRN("could not upload file(%s): result=%d", path, result);
|
||||
FdManager::get()->Close(ent);
|
||||
return result;
|
||||
@ -2001,7 +2026,7 @@ static int s3fs_flush(const char* path, struct fuse_file_info* fi)
|
||||
meta["x-amz-meta-mtime"] = str(ent_mtime);
|
||||
}
|
||||
}
|
||||
result = ent->Flush(meta, true, false);
|
||||
result = ent->Flush(meta, false);
|
||||
FdManager::get()->Close(ent);
|
||||
}
|
||||
S3FS_MALLOCTRIM(0);
|
||||
@ -2077,9 +2102,23 @@ static S3fsCurl* multi_head_retry_callback(S3fsCurl* s3fscurl)
|
||||
if(!s3fscurl){
|
||||
return NULL;
|
||||
}
|
||||
int ssec_key_pos = s3fscurl->GetLastPreHeadSeecKeyPos();
|
||||
int next_retry_count = s3fscurl->GetMultipartRetryCount() + 1;
|
||||
|
||||
if(s3fscurl->IsOverMultipartRetryCount()){
|
||||
DPRN("Over retry count(%d) limit(%s).", s3fscurl->GetMultipartRetryCount(), s3fscurl->GetSpacialSavedPath().c_str());
|
||||
return NULL;
|
||||
if(S3fsCurl::IsSseCustomMode()){
|
||||
// If sse-c mode, start check not sse-c(ssec_key_pos = -1).
|
||||
// do increment ssec_key_pos for checking all sse-c key.
|
||||
next_retry_count = 0;
|
||||
ssec_key_pos++;
|
||||
if(S3fsCurl::GetSseKeyCount() <= ssec_key_pos){
|
||||
DPRN("Over retry count(%d) limit(%s).", s3fscurl->GetMultipartRetryCount(), s3fscurl->GetSpacialSavedPath().c_str());
|
||||
return NULL;
|
||||
}
|
||||
}else{
|
||||
DPRN("Over retry count(%d) limit(%s).", s3fscurl->GetMultipartRetryCount(), s3fscurl->GetSpacialSavedPath().c_str());
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
S3fsCurl* newcurl = new S3fsCurl(s3fscurl->IsUseAhbe());
|
||||
@ -2087,12 +2126,12 @@ static S3fsCurl* multi_head_retry_callback(S3fsCurl* s3fscurl)
|
||||
string base_path = s3fscurl->GetBasePath();
|
||||
string saved_path = s3fscurl->GetSpacialSavedPath();
|
||||
|
||||
if(!newcurl->PreHeadRequest(path, base_path, saved_path)){
|
||||
if(!newcurl->PreHeadRequest(path, base_path, saved_path, ssec_key_pos)){
|
||||
DPRN("Could not duplicate curl object(%s).", saved_path.c_str());
|
||||
delete newcurl;
|
||||
return NULL;
|
||||
}
|
||||
newcurl->SetMultipartRetryCount(s3fscurl->GetMultipartRetryCount() + 1);
|
||||
newcurl->SetMultipartRetryCount(next_retry_count);
|
||||
|
||||
return newcurl;
|
||||
}
|
||||
@ -2134,6 +2173,8 @@ static int readdir_multi_head(const char* path, S3ObjList& head, void* buf, fuse
|
||||
continue;
|
||||
}
|
||||
|
||||
// First check for directory, start checking "not sse-c".
|
||||
// If checking failed, retry to check with "sse-c" by retry callback func when sse-c mode.
|
||||
S3fsCurl* s3fscurl = new S3fsCurl();
|
||||
if(!s3fscurl->PreHeadRequest(disppath, (*iter), disppath)){ // target path = cache key path.(ex "dir/")
|
||||
DPRNNN("Could not make curl object for head request(%s).", disppath.c_str());
|
||||
@ -2271,8 +2312,20 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter)
|
||||
next_marker = (char*)tmpch;
|
||||
xmlFree(tmpch);
|
||||
}else{
|
||||
DPRN("Could not find next marker, thus break loop.");
|
||||
truncated = false;
|
||||
// If did not specify "delimiter", s3 did not return "NextMarker".
|
||||
// On this case, can use lastest name for next marker.
|
||||
//
|
||||
string lastname;
|
||||
if(!head.GetLastName(lastname)){
|
||||
DPRN("Could not find next marker, thus break loop.");
|
||||
truncated = false;
|
||||
}else{
|
||||
next_marker = s3_realpath.substr(1);
|
||||
if(0 == s3_realpath.length() || '/' != s3_realpath[s3_realpath.length() - 1]){
|
||||
next_marker += "/";
|
||||
}
|
||||
next_marker += lastname;
|
||||
}
|
||||
}
|
||||
}
|
||||
S3FS_XMLFREEDOC(doc);
|
||||
@ -2595,7 +2648,13 @@ static int remote_mountpath_exists(const char* path)
|
||||
static void* s3fs_init(struct fuse_conn_info* conn)
|
||||
{
|
||||
FPRN("init");
|
||||
LOWSYSLOGPRINT(LOG_ERR, "init $Rev$");
|
||||
LOWSYSLOGPRINT(LOG_ERR, "init v%s (%s)", VERSION, s3fs_crypt_lib_name());
|
||||
|
||||
// ssl init
|
||||
if(!s3fs_init_global_ssl()){
|
||||
fprintf(stderr, "%s: could not initialize for ssl libraries.\n", program_name.c_str());
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
// init curl
|
||||
if(!S3fsCurl::InitS3fsCurl("/etc/mime.types")){
|
||||
@ -2637,6 +2696,8 @@ static void s3fs_destroy(void*)
|
||||
if(is_remove_cache && !FdManager::DeleteCacheDirectory()){
|
||||
DPRN("Could not remove cache directory.");
|
||||
}
|
||||
// ssl
|
||||
s3fs_destroy_global_ssl();
|
||||
}
|
||||
|
||||
static int s3fs_access(const char* path, int mask)
|
||||
@ -2840,10 +2901,17 @@ static int s3fs_utility_mode(void)
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
// ssl init
|
||||
if(!s3fs_init_global_ssl()){
|
||||
fprintf(stderr, "%s: could not initialize for ssl libraries.\n", program_name.c_str());
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
// init curl
|
||||
if(!S3fsCurl::InitS3fsCurl("/etc/mime.types")){
|
||||
fprintf(stderr, "%s: Could not initiate curl library.\n", program_name.c_str());
|
||||
LOWSYSLOGPRINT(LOG_ERR, "Could not initiate curl library.");
|
||||
s3fs_destroy_global_ssl();
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
@ -2888,6 +2956,10 @@ static int s3fs_utility_mode(void)
|
||||
if(!S3fsCurl::DestroyS3fsCurl()){
|
||||
DPRN("Could not release curl library.");
|
||||
}
|
||||
|
||||
// ssl
|
||||
s3fs_destroy_global_ssl();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -2957,10 +3029,18 @@ static int check_for_aws_format(void)
|
||||
ifstream PF(passwd_file.c_str());
|
||||
if(PF.good()){
|
||||
while (getline(PF, line)){
|
||||
if(line[0]=='#')
|
||||
if(line[0]=='#'){
|
||||
continue;
|
||||
if(line.size() == 0)
|
||||
}
|
||||
if(line.size() == 0){
|
||||
continue;
|
||||
}
|
||||
if('\r' == line[line.size() - 1]){
|
||||
line = line.substr(0, line.size() - 1);
|
||||
if(line.size() == 0){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
first_pos = line.find_first_of(" \t");
|
||||
if(first_pos != string::npos){
|
||||
@ -3110,6 +3190,12 @@ static int read_passwd_file(void)
|
||||
if(line.size() == 0){
|
||||
continue;
|
||||
}
|
||||
if('\r' == line[line.size() - 1]){
|
||||
line = line.substr(0, line.size() - 1);
|
||||
if(line.size() == 0){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
first_pos = line.find_first_of(" \t");
|
||||
if(first_pos != string::npos){
|
||||
@ -3476,22 +3562,44 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
return 0;
|
||||
}
|
||||
if(0 == strcmp(arg, "use_sse") || 0 == STR2NCMP(arg, "use_sse=")){
|
||||
off_t sse = 1;
|
||||
// for an old format.
|
||||
if(0 == STR2NCMP(arg, "use_sse=")){
|
||||
sse = s3fs_strtoofft(strchr(arg, '=') + sizeof(char));
|
||||
}
|
||||
if(0 == sse){
|
||||
S3fsCurl::SetUseSse(false);
|
||||
}else if(1 == sse){
|
||||
if(S3fsCurl::GetUseRrs()){
|
||||
fprintf(stderr, "%s: use_sse option could not be specified with use_rrs.\n", program_name.c_str());
|
||||
return -1;
|
||||
}
|
||||
S3fsCurl::SetUseSse(true);
|
||||
const char* ssecfile = &arg[strlen("use_sse=")];
|
||||
if(0 == strcmp(ssecfile, "1")){
|
||||
if(S3fsCurl::IsSseCustomMode()){
|
||||
fprintf (stderr, "%s: already set SSE-C key by environment, and confrict use_sse option.\n", program_name.c_str());
|
||||
return -1;
|
||||
}
|
||||
S3fsCurl::SetUseSse(true);
|
||||
}else{
|
||||
// testing sse-c, try to load AES256 keys
|
||||
struct stat st;
|
||||
if(0 != stat(ssecfile, &st)){
|
||||
fprintf (stderr, "%s: could not open use_sse keys file(%s)\n", program_name.c_str(), ssecfile);
|
||||
return -1;
|
||||
}
|
||||
if(st.st_mode & (S_IXUSR | S_IRWXG | S_IRWXO)){
|
||||
fprintf (stderr, "%s: use_sse keys file %s should be 0600 permissions\n", program_name.c_str(), ssecfile);
|
||||
return -1;
|
||||
}
|
||||
if(!S3fsCurl::SetSseKeys(ssecfile)){
|
||||
fprintf (stderr, "%s: failed to load use_sse keys file %s\n", program_name.c_str(), ssecfile);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}else{
|
||||
fprintf(stderr, "%s: poorly formed argument to option: use_sse\n", program_name.c_str());
|
||||
return -1;
|
||||
if(S3fsCurl::GetUseRrs()){
|
||||
fprintf(stderr, "%s: use_sse option could not be specified with use_rrs.\n", program_name.c_str());
|
||||
return -1;
|
||||
}
|
||||
if(S3fsCurl::IsSseCustomMode()){
|
||||
fprintf (stderr, "%s: already set SSE-C key by environment, and confrict use_sse option.\n", program_name.c_str());
|
||||
return -1;
|
||||
}
|
||||
S3fsCurl::SetUseSse(true);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@ -3641,6 +3749,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
if(0 == strcmp(arg, "use_path_request_style")){
|
||||
pathrequeststyle = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// debug option
|
||||
//
|
||||
@ -3740,6 +3852,9 @@ int main(int argc, char* argv[])
|
||||
}
|
||||
}
|
||||
|
||||
// Load SSE-C Key from env
|
||||
S3fsCurl::LoadEnvSseKeys();
|
||||
|
||||
// clear this structure
|
||||
memset(&s3fs_oper, 0, sizeof(s3fs_oper));
|
||||
|
||||
@ -3872,15 +3987,17 @@ int main(int argc, char* argv[])
|
||||
s3fs_oper.access = s3fs_access;
|
||||
s3fs_oper.create = s3fs_create;
|
||||
|
||||
// init NSS
|
||||
S3FS_INIT_NSS();
|
||||
if(!s3fs_init_global_ssl()){
|
||||
fprintf(stderr, "%s: could not initialize for ssl libraries.\n", program_name.c_str());
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
// now passing things off to fuse, fuse will finish evaluating the command line args
|
||||
fuse_res = fuse_main(custom_args.argc, custom_args.argv, &s3fs_oper, NULL);
|
||||
fuse_opt_free_args(&custom_args);
|
||||
|
||||
// cleanup NSS
|
||||
S3FS_CLEANUP_NSS();
|
||||
s3fs_destroy_global_ssl();
|
||||
|
||||
// cleanup xml2
|
||||
xmlCleanupParser();
|
||||
S3FS_MALLOCTRIM(0);
|
||||
@ -3888,3 +4005,11 @@ int main(int argc, char* argv[])
|
||||
exit(fuse_res);
|
||||
}
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
|
||||
60
src/s3fs.h
60
src/s3fs.h
@ -1,3 +1,22 @@
|
||||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
#ifndef S3FS_S3_H_
|
||||
#define S3FS_S3_H_
|
||||
|
||||
@ -65,36 +84,15 @@
|
||||
|
||||
#endif // HAVE_MALLOC_TRIM
|
||||
|
||||
//
|
||||
// For initializing libcurl with NSS
|
||||
// Normally libcurl initializes the NSS library, but usually allows
|
||||
// you to initialize s3fs forcibly. Because Memory leak is reported
|
||||
// in valgrind(about curl_global_init() function), and this is for
|
||||
// the cancellation. When "--enable-nss-init" option is specified
|
||||
// at configurarion, it makes NSS_INIT_ENABLED flag into Makefile.
|
||||
// NOTICE
|
||||
// This defines and macros is temporary, and this should be deleted.
|
||||
//
|
||||
#ifdef NSS_INIT_ENABLED
|
||||
#include <nss.h>
|
||||
#include <prinit.h>
|
||||
|
||||
#define S3FS_INIT_NSS() \
|
||||
{ \
|
||||
NSS_NoDB_Init(NULL); \
|
||||
}
|
||||
#define S3FS_CLEANUP_NSS() \
|
||||
{ \
|
||||
NSS_Shutdown(); \
|
||||
PL_ArenaFinish(); \
|
||||
PR_Cleanup(); \
|
||||
}
|
||||
|
||||
#else // NSS_INIT_ENABLED
|
||||
|
||||
#define S3FS_INIT_NSS()
|
||||
#define S3FS_CLEANUP_NSS()
|
||||
|
||||
#endif // NSS_INIT_ENABLED
|
||||
char* get_object_sseckey_md5(const char* path);
|
||||
|
||||
#endif // S3FS_S3_H_
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
|
||||
54
src/s3fs_auth.h
Normal file
54
src/s3fs_auth.h
Normal file
@ -0,0 +1,54 @@
|
||||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
#ifndef S3FS_AUTH_H_
|
||||
#define S3FS_AUTH_H_
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility functions for Authentication
|
||||
//-------------------------------------------------------------------
|
||||
//
|
||||
// in common_auth.cpp
|
||||
//
|
||||
char* s3fs_base64(unsigned char* input, size_t length);
|
||||
std::string s3fs_get_content_md5(int fd);
|
||||
std::string s3fs_md5sum(int fd, off_t start, ssize_t size);
|
||||
|
||||
//
|
||||
// in xxxxxx_auth.cpp
|
||||
//
|
||||
const char* s3fs_crypt_lib_name(void);
|
||||
bool s3fs_init_global_ssl(void);
|
||||
bool s3fs_destroy_global_ssl(void);
|
||||
bool s3fs_init_crypt_mutex(void);
|
||||
bool s3fs_destroy_crypt_mutex(void);
|
||||
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen);
|
||||
size_t get_md5_digest_length(void);
|
||||
unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size);
|
||||
|
||||
#endif // S3FS_AUTH_H_
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
@ -41,6 +41,7 @@
|
||||
#include "s3fs_util.h"
|
||||
#include "string_util.h"
|
||||
#include "s3fs.h"
|
||||
#include "s3fs_auth.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@ -228,6 +229,26 @@ bool S3ObjList::IsDir(const char* name) const
|
||||
return ps3obj->is_dir;
|
||||
}
|
||||
|
||||
bool S3ObjList::GetLastName(std::string& lastname) const
|
||||
{
|
||||
bool result = false;
|
||||
lastname = "";
|
||||
for(s3obj_t::const_iterator iter = objects.begin(); iter != objects.end(); iter++){
|
||||
if((*iter).second.orgname.length()){
|
||||
if(0 > strcmp(lastname.c_str(), (*iter).second.orgname.c_str())){
|
||||
lastname = (*iter).second.orgname;
|
||||
result = true;
|
||||
}
|
||||
}else{
|
||||
if(0 > strcmp(lastname.c_str(), (*iter).second.normalname.c_str())){
|
||||
lastname = (*iter).second.normalname;
|
||||
result = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
bool S3ObjList::GetNameList(s3obj_list_t& list, bool OnlyNormalized, bool CutSlash) const
|
||||
{
|
||||
s3obj_t::const_iterator iter;
|
||||
@ -311,7 +332,7 @@ MVNODE *create_mvnode(const char *old_path, const char *new_path, bool is_dir, b
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if(NULL == (p_new_path = strdup(new_path))){
|
||||
if(NULL == (p_new_path = strdup(new_path))){
|
||||
free(p);
|
||||
free(p_old_path);
|
||||
printf("create_mvnode: could not allocation memory for p_new_path\n");
|
||||
@ -329,7 +350,7 @@ MVNODE *create_mvnode(const char *old_path, const char *new_path, bool is_dir, b
|
||||
}
|
||||
|
||||
//
|
||||
// Add sorted MVNODE data(Ascending order)
|
||||
// Add sorted MVNODE data(Ascending order)
|
||||
//
|
||||
MVNODE *add_mvnode(MVNODE** head, MVNODE** tail, const char *old_path, const char *new_path, bool is_dir, bool normdir)
|
||||
{
|
||||
@ -670,9 +691,9 @@ mode_t get_mode(headers_t& meta, const char* path, bool checkdir, bool forcedir)
|
||||
}
|
||||
}
|
||||
// Checking the bitmask, if the last 3 bits are all zero then process as a regular
|
||||
// file type (S_IFDIR or S_IFREG), otherwise return mode unmodified so that S_IFIFO,
|
||||
// file type (S_IFDIR or S_IFREG), otherwise return mode unmodified so that S_IFIFO,
|
||||
// S_IFSOCK, S_IFCHR, S_IFLNK and S_IFBLK devices can be processed properly by fuse.
|
||||
if(!(mode & S_IFMT)){
|
||||
if(!(mode & S_IFMT)){
|
||||
if(!isS3sync){
|
||||
if(checkdir){
|
||||
if(forcedir){
|
||||
@ -822,7 +843,7 @@ void show_usage (void)
|
||||
void show_help (void)
|
||||
{
|
||||
show_usage();
|
||||
printf(
|
||||
printf(
|
||||
"\n"
|
||||
"Mount an Amazon S3 bucket as a file system.\n"
|
||||
"\n"
|
||||
@ -854,7 +875,24 @@ void show_help (void)
|
||||
" - this option makes Amazon's Reduced Redundancy Storage enable.\n"
|
||||
"\n"
|
||||
" use_sse (default is disable)\n"
|
||||
" - this option makes Amazon's Server Site Encryption enable.\n"
|
||||
" - use Amazon<EFBFBD>fs Server-Site Encryption or Server-Side Encryption\n"
|
||||
" with Customer-Provided Encryption Keys.\n"
|
||||
" this option can not be specified with use_rrs. specifying only \n"
|
||||
" \"use_sse\" or \"use_sse=1\" enables Server-Side Encryption.\n"
|
||||
" (use_sse=1 for old version)\n"
|
||||
" specifying this option with file path which has some SSE-C\n"
|
||||
" secret key enables Server-Side Encryption with Customer-Provided\n"
|
||||
" Encryption Keys.(use_sse=file)\n"
|
||||
" the file must be 600 permission. the file can have some lines,\n"
|
||||
" each line is one SSE-C key. the first line in file is used as\n"
|
||||
" Customer-Provided Encryption Keys for uploading and changing\n"
|
||||
" headers etc.\n"
|
||||
" if there are some keys after first line, those are used\n"
|
||||
" downloading object which are encripted by not first key.\n"
|
||||
" so that, you can keep all SSE-C keys in file, that is SSE-C\n"
|
||||
" key history.\n"
|
||||
" if AWSSSECKEYS environment is set, you can set SSE-C key instead\n"
|
||||
" of this option.\n"
|
||||
"\n"
|
||||
" public_bucket (default=\"\" which means disabled)\n"
|
||||
" - anonymously mount a public bucket when set to 1\n"
|
||||
@ -966,6 +1004,11 @@ void show_help (void)
|
||||
" only rename command(ex. mv). If this option is specified with\n"
|
||||
" nocopapi, the s3fs ignores it.\n"
|
||||
"\n"
|
||||
" use_path_request_style (use legacy API calling style)\n"
|
||||
" Enble compatibility with S3-like APIs which do not support\n"
|
||||
" the virtual-host request style, by using the older path request\n"
|
||||
" style.\n"
|
||||
"\n"
|
||||
"FUSE/mount Options:\n"
|
||||
"\n"
|
||||
" Most of the generic mount options described in 'man mount' are\n"
|
||||
@ -997,11 +1040,20 @@ void show_help (void)
|
||||
void show_version(void)
|
||||
{
|
||||
printf(
|
||||
"Amazon Simple Storage Service File System %s\n"
|
||||
"Amazon Simple Storage Service File System V%s with %s\n"
|
||||
"Copyright (C) 2010 Randy Rizun <rrizun@gmail.com>\n"
|
||||
"License GPL2: GNU GPL version 2 <http://gnu.org/licenses/gpl.html>\n"
|
||||
"This is free software: you are free to change and redistribute it.\n"
|
||||
"There is NO WARRANTY, to the extent permitted by law.\n", VERSION );
|
||||
"There is NO WARRANTY, to the extent permitted by law.\n",
|
||||
VERSION, s3fs_crypt_lib_name());
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
|
||||
@ -1,3 +1,22 @@
|
||||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
#ifndef S3FS_S3FS_UTIL_H_
|
||||
#define S3FS_S3FS_UTIL_H_
|
||||
|
||||
@ -51,6 +70,7 @@ class S3ObjList
|
||||
std::string GetETag(const char* name) const;
|
||||
bool IsDir(const char* name) const;
|
||||
bool GetNameList(s3obj_list_t& list, bool OnlyNormalized = true, bool CutSlash = true) const;
|
||||
bool GetLastName(std::string& lastname) const;
|
||||
|
||||
static bool MakeHierarchizedList(s3obj_list_t& list, bool haveSlash);
|
||||
};
|
||||
@ -116,3 +136,12 @@ void show_help(void);
|
||||
void show_version(void);
|
||||
|
||||
#endif // S3FS_S3FS_UTIL_H_
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
|
||||
@ -186,8 +186,18 @@ string prepare_url(const char* url)
|
||||
uri_length = 8;
|
||||
}
|
||||
uri = url_str.substr(0, uri_length);
|
||||
host = bucket + "." + url_str.substr(uri_length, bucket_pos - uri_length).c_str();
|
||||
path = url_str.substr((bucket_pos + bucket_length));
|
||||
|
||||
if(!pathrequeststyle){
|
||||
host = bucket + "." + url_str.substr(uri_length, bucket_pos - uri_length).c_str();
|
||||
path = url_str.substr((bucket_pos + bucket_length));
|
||||
}else{
|
||||
host = url_str.substr(uri_length, bucket_pos - uri_length).c_str();
|
||||
string part = url_str.substr((bucket_pos + bucket_length));
|
||||
if('/' != part[0]){
|
||||
part = "/" + part;
|
||||
}
|
||||
path = "/" + bucket + part;
|
||||
}
|
||||
|
||||
url_str = uri + host + path;
|
||||
|
||||
@ -208,3 +218,11 @@ string get_date()
|
||||
return buf;
|
||||
}
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
|
||||
@ -1,3 +1,22 @@
|
||||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
#ifndef S3FS_STRING_UTIL_H_
|
||||
#define S3FS_STRING_UTIL_H_
|
||||
|
||||
@ -32,3 +51,12 @@ std::string prepare_url(const char* url);
|
||||
bool get_keyword_value(std::string& target, const char* keyword, std::string& value);
|
||||
|
||||
#endif // S3FS_STRING_UTIL_H_
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: noet sw=4 ts=4 fdm=marker
|
||||
* vim<600: noet sw=4 ts=4
|
||||
*/
|
||||
|
||||
@ -1,3 +1,22 @@
|
||||
######################################################################
|
||||
# s3fs - FUSE-based file system backed by Amazon S3
|
||||
#
|
||||
# Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
######################################################################
|
||||
TESTS=small-integration-test.sh
|
||||
|
||||
EXTRA_DIST = \
|
||||
|
||||
Reference in New Issue
Block a user