youngkingdom/s3fs-fuse
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Summary of Changes(1.65 -> 1.66)

Browse Source 
==========================
List of Changes
==========================
1) Fixes bugs
    Fixes Issue 321: "no write permission for non-root user".
    (http://code.google.com/p/s3fs/issues/detail?id=321)
    Fixes a bug which s3fs does not set uid/gid headers when making symlink.

2) Cleanup  code.
    Adds a common function which  converts the Last-Modified header to utime.
    Deletes the useless cord and arranged it.

3) xmlns
    Changes that s3fs can decide using the xmlns url automatically.
    Then the noxmlns option is not needed anymore, but it is left.

4) Changes cache for performance
    Changes stat cache, it accumulates stat information and some headers.
    By adding some headers into cache, s3fs does not need to call curl_get_headers function.
    After changing, one cache entry increases in about 500 bytes from about 144 byte.
    
    Adds one condition to get out of the cache, that condition is by looking object's ETag.
    It works good for noticing changes about obojects.




git-svn-id: http://s3fs.googlecode.com/svn/trunk@400 df820570-a93a-0410-bd06-b72b767a4274
This commit is contained in:
ggtakec@gmail.com
2013-04-06 17:39:22 +00:00
parent a35cdc73b7
commit 8bd1483374
9 changed files with 751 additions and 412 deletions
Download Patch File Download Diff File Expand all files Collapse all files

329
src/cache.cpp
View File

@ -24,141 +24,324 @@
#include <unistd.h>
#include <pthread.h>
#include <string.h>
#include <iostream>
#include <assert.h>
#include <string>
#include <map>
#include <algorithm>
#include "common.h"
#include "cache.h"
#include "s3fs_util.h"
using namespace std;
//-------------------------------------------------------------------
// Typedef
// Static
//-------------------------------------------------------------------
typedef std::map<std::string, struct stat_cache_entry> stat_cache_t; // key=path
StatCache StatCache::singleton;
pthread_mutex_t StatCache::stat_cache_lock;
//-------------------------------------------------------------------
// Static valiables
// Constructor/Destructor
//-------------------------------------------------------------------
static stat_cache_t stat_cache;
static pthread_mutex_t stat_cache_lock;
//-------------------------------------------------------------------
// Functions
//-------------------------------------------------------------------
int init_stat_cache_mutex(void)
StatCache::StatCache()
{
return pthread_mutex_init(&stat_cache_lock, NULL);
if(this == StatCache::getStatCacheData()){
pthread_mutex_init(&(StatCache::stat_cache_lock), NULL);
}else{
assert(false);
}
CacheSize = 1000;
ExpireTime = 0;
IsExpireTime = false;
}
int destroy_stat_cache_mutex(void)
StatCache::~StatCache()
{
return pthread_mutex_destroy(&stat_cache_lock);
if(this == StatCache::getStatCacheData()){
pthread_mutex_destroy(&(StatCache::stat_cache_lock));
}else{
assert(false);
}
}
int get_stat_cache_entry(const char *path, struct stat *buf)
//-------------------------------------------------------------------
// Methods
//-------------------------------------------------------------------
unsigned long StatCache::GetCacheSize(void) const
{
int is_delete_cache = 0;
string strpath = path;
return CacheSize;
}
pthread_mutex_lock(&stat_cache_lock);
unsigned long StatCache::SetCacheSize(unsigned long size)
{
unsigned long old = CacheSize;
CacheSize = size;
return old;
}
time_t StatCache::GetExpireTime(void) const
{
return (IsExpireTime ? ExpireTime : (-1));
}
time_t StatCache::SetExpireTime(time_t expire)
{
time_t old = ExpireTime;
ExpireTime = expire;
IsExpireTime = true;
return old;
}
time_t StatCache::UnsetExpireTime(void)
{
time_t old = IsExpireTime ? ExpireTime : (-1);
ExpireTime = 0;
IsExpireTime = false;
return old;
}
bool StatCache::GetStat(string& key, struct stat* pst, headers_t* meta, bool overcheck, const char* petag)
{
bool is_delete_cache = false;
string strpath = key;
pthread_mutex_lock(&StatCache::stat_cache_lock);
stat_cache_t::iterator iter = stat_cache.end();
if('/' != strpath[strpath.length() - 1]){
if(overcheck && '/' != strpath[strpath.length() - 1]){
strpath += "/";
iter = stat_cache.find(strpath.c_str());
}
if(iter == stat_cache.end()){
strpath = path;
strpath = key;
iter = stat_cache.find(strpath.c_str());
}
if(iter != stat_cache.end()) {
if(!is_stat_cache_expire_time || ((*iter).second.cache_date + stat_cache_expire_time) >= time(NULL)){
// hit
FGPRINT(" stat cache hit [path=%s] [time=%ld] [hit count=%lu]\n",
strpath.c_str(), (*iter).second.cache_date, (*iter).second.hit_count);
if(buf != NULL){
*buf = (*iter).second.stbuf;
if(!IsExpireTime|| ((*iter).second.cache_date + ExpireTime) >= time(NULL)){
// hit without checking etag
if(petag){
string stretag = (*iter).second.meta["ETag"];
if('\0' != petag[0] && 0 != strcmp(petag, stretag.c_str())){
is_delete_cache = true;
}
}
(*iter).second.hit_count++;
pthread_mutex_unlock(&stat_cache_lock);
return 0;
if(is_delete_cache){
// not hit by different ETag
FGPRINT(" stat cache not hit by ETag[path=%s][time=%ld][hit count=%lu][ETag(%s)!=(%s)]\n",
strpath.c_str(), (*iter).second.cache_date, (*iter).second.hit_count,
petag ? petag : "null", (*iter).second.meta["ETag"].c_str());
}else{
// hit
FGPRINT(" stat cache hit [path=%s] [time=%ld] [hit count=%lu]\n",
strpath.c_str(), (*iter).second.cache_date, (*iter).second.hit_count);
if(pst!= NULL){
*pst= (*iter).second.stbuf;
}
if(meta != NULL){
meta->clear();
(*meta) = (*iter).second.meta;
}
(*iter).second.hit_count++;
pthread_mutex_unlock(&StatCache::stat_cache_lock);
return true;
}
}else{
// timeout
is_delete_cache = 1;
is_delete_cache = true;
}
}
pthread_mutex_unlock(&stat_cache_lock);
pthread_mutex_unlock(&StatCache::stat_cache_lock);
if(is_delete_cache){
delete_stat_cache_entry(strpath.c_str());
DelStat(strpath);
}
return -1;
return false;
}
void add_stat_cache_entry(const char *path, struct stat *st)
bool StatCache::AddStat(std::string& key, headers_t& meta)
{
FGPRINT(" add_stat_cache_entry[path=%s]\n", path);
if(CacheSize< 1){
return true;
}
FGPRINT(" add_stat_cache_entry[path=%s]\n", key.c_str());
if(max_stat_cache_size < 1){
return;
if(stat_cache.size() > CacheSize){
if(!TruncateCache()){
return false;
}
}
if(stat_cache.size() > max_stat_cache_size){
truncate_stat_cache();
struct stat st;
if(!convert_header_to_stat(key.c_str(), meta, &st)){
return false;
}
pthread_mutex_lock(&stat_cache_lock);
stat_cache[path].stbuf = *st;
stat_cache[path].cache_date = time(NULL); // Set time.
pthread_mutex_unlock(&stat_cache_lock);
pthread_mutex_lock(&StatCache::stat_cache_lock);
stat_cache[key].stbuf = st;
stat_cache[key].hit_count = 0;
stat_cache[key].cache_date = time(NULL); // Set time.
//copy only some keys
for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) {
string tag = (*iter).first;
string value = (*iter).second;
if(tag == "Content-Type"){
stat_cache[key].meta[tag] = value;
}else if(tag == "Content-Length"){
stat_cache[key].meta[tag] = value;
}else if(tag == "ETag"){
stat_cache[key].meta[tag] = value;
}else if(tag == "Last-Modified"){
stat_cache[key].meta[tag] = value;
}else if(tag.substr(0, 5) == "x-amz"){
stat_cache[key].meta[tag] = value;
}else{
// Check for upper case
transform(tag.begin(), tag.end(), tag.begin(), static_cast<int (*)(int)>(std::tolower));
if(tag.substr(0, 5) == "x-amz"){
stat_cache[key].meta[tag] = value;
}
}
}
pthread_mutex_unlock(&StatCache::stat_cache_lock);
return true;
}
void delete_stat_cache_entry(const char *path)
bool StatCache::TruncateCache(void)
{
FGPRINT(" delete_stat_cache_entry[path=%s]\n", path);
string path_to_delete;
unsigned int lowest_hit_count = 0;
pthread_mutex_lock(&stat_cache_lock);
stat_cache_t::iterator iter = stat_cache.find(path);
pthread_mutex_lock(&StatCache::stat_cache_lock);
stat_cache_t::iterator iter;
for(iter = stat_cache.begin(); iter != stat_cache.end(); iter++) {
if(!lowest_hit_count) {
lowest_hit_count = (*iter).second.hit_count;
path_to_delete = (*iter).first;
}
if(lowest_hit_count > (*iter).second.hit_count){
lowest_hit_count = (*iter).second.hit_count;
path_to_delete = (*iter).first;
}
}
stat_cache.erase(path_to_delete);
pthread_mutex_unlock(&StatCache::stat_cache_lock);
FGPRINT(" truncate_stat_cache_entry[path=%s]\n", path_to_delete.c_str());
return true;
}
bool StatCache::DelStat(const char* key)
{
if(!key){
return false;
}
FGPRINT(" delete_stat_cache_entry[path=%s]\n", key);
pthread_mutex_lock(&StatCache::stat_cache_lock);
stat_cache_t::iterator iter = stat_cache.find(key);
if(iter != stat_cache.end()){
stat_cache.erase(iter);
}
if(0 < strlen(path) && '/' != path[strlen(path) - 1]){
// If there is "path/" cache, delete it.
string strpath = path;
strpath += "/";
if(0 < strlen(key) && 0 != strcmp(key, "/")){
string strpath = key;
if('/' == strpath[strpath.length() - 1]){
// If there is "path" cache, delete it.
strpath = strpath.substr(0, strpath.length() - 1);
}else{
// If there is "path/" cache, delete it.
strpath += "/";
}
iter = stat_cache.find(strpath.c_str());
if(iter != stat_cache.end()){
stat_cache.erase(iter);
}
}
pthread_mutex_unlock(&stat_cache_lock);
pthread_mutex_unlock(&StatCache::stat_cache_lock);
return true;
}
void truncate_stat_cache() {
string path_to_delete;
unsigned int hit_count = 0;
unsigned int lowest_hit_count = 0;
//-------------------------------------------------------------------
// Functions
//-------------------------------------------------------------------
bool convert_header_to_stat(const char* path, headers_t& meta, struct stat* pst)
{
headers_t::const_iterator iter;
pthread_mutex_lock(&stat_cache_lock);
stat_cache_t::iterator iter;
for(iter = stat_cache.begin(); iter != stat_cache.end(); iter++) {
hit_count = (* iter).second.hit_count;
if(!path || !pst){
return false;
}
memset(pst, 0, sizeof(struct stat));
if(!lowest_hit_count) {
lowest_hit_count = hit_count;
path_to_delete = (* iter).first;
}
if(lowest_hit_count > hit_count){
path_to_delete = (* iter).first;
}
pst->st_nlink = 1; // see fuse FAQ
// mode
iter = meta.find("x-amz-meta-mode");
if(iter != meta.end()){
pst->st_mode = get_mode((*iter).second.c_str());
}
stat_cache.erase(path_to_delete);
pthread_mutex_unlock(&stat_cache_lock);
// content-type
string strConType;
iter = meta.find("Content-Type");
if(iter != meta.end()){
strConType = (*iter).second;
}
if(strConType == "application/x-directory"){
pst->st_mode |= S_IFDIR;
}else if(0 < strlen(path) && '/' == path[strlen(path) - 1]){
if(strConType == "binary/octet-stream" || strConType == "application/octet-stream"){
pst->st_mode |= S_IFDIR;
}else{
pst->st_mode |= S_IFREG;
}
}else{
pst->st_mode |= S_IFREG;
}
FGPRINT(" purged %s from the stat cache\n", path_to_delete.c_str());
// blocks
if(S_ISREG(pst->st_mode)){
pst->st_blocks = get_blocks(pst->st_size);
}
// mtime
iter = meta.find("x-amz-meta-mtime");
if(iter != meta.end()){
pst->st_mtime = get_mtime((*iter).second.c_str());
}
if(pst->st_mtime == 0) {
iter = meta.find("Last-Modified");
if(iter != meta.end()){
pst->st_mtime = get_lastmodified((*iter).second.c_str());
}
}
if(-1 == pst->st_mtime){
pst->st_mtime = 0;
}
// size
iter = meta.find("Content-Length");
if(iter != meta.end()){
pst->st_size = get_size((*iter).second.c_str());
}
// uid/gid
iter = meta.find("x-amz-meta-uid");
if(iter != meta.end()){
pst->st_uid = get_uid((*iter).second.c_str());
}
iter = meta.find("x-amz-meta-gid");
if(iter != meta.end()){
pst->st_gid = get_gid((*iter).second.c_str());
}
return true;
}