table_t(headers_t headers, formatters_t formatters, alignments_t alignments, size_t sort_index = 0, sort_direction direction = sort_direction::ascending)
: formatters(formatters), alignments(alignments), sort_index(sort_index), direction(direction) {
std::transform(headers.begin(), headers.end(), this->headers.begin(), [](auto& x) {
return to_upper(x);
});
update_column_max_width(headers);
}
//
// Returns it whether it is an object with need checking in detail.
// If this function returns true, the object is possible to be directory
// and is needed checking detail(searching sub object).
//
bool is_need_check_obj_detail(headers_t& meta)
{
headers_t::const_iterator iter;
// directory object is Content-Length as 0.
if(0 != get_size(meta)){
return false;
}
// if the object has x-amz-meta information, checking is no more.
if(meta.end() != meta.find("x-amz-meta-mode") ||
meta.end() != meta.find("x-amz-meta-mtime") ||
meta.end() != meta.find("x-amz-meta-uid") ||
meta.end() != meta.find("x-amz-meta-gid") ||
meta.end() != meta.find("x-amz-meta-owner") ||
meta.end() != meta.find("x-amz-meta-group") ||
meta.end() != meta.find("x-amz-meta-permissions") )
{
return false;
}
// if there is not Content-Type, or Content-Type is "x-directory",
// checking is no more.
if(meta.end() == (iter = find_content_type(meta))){
return false;
}
if("application/x-directory" == (*iter).second){
return false;
}
return true;
}
/**
* create or update s3 object
* @return fuse return code
*/
static int
put_local_fd(const char* path, headers_t meta, int fd) {
string resource = urlEncode(service_path + bucket + path);
string url = host + resource;
struct stat st;
if (fstat(fd, &st) == -1)
Yikes(-errno);
auto_curl curl;
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
string responseText;
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseText);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback);
curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(st.st_size)); // Content-Length
FILE* f = fdopen(fd, "rb");
if (f == 0)
Yikes(-errno);
curl_easy_setopt(curl, CURLOPT_INFILE, f);
string ContentType = meta["Content-Type"];
auto_curl_slist headers;
string date = get_date();
headers.append("Date: "+date);
meta["x-amz-acl"] = default_acl;
for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) {
string key = (*iter).first;
string value = (*iter).second;
if (key == "Content-Type")
headers.append(key+":"+value);
if (key.substr(0,9) == "x-amz-acl")
headers.append(key+":"+value);
if (key.substr(0,10) == "x-amz-meta")
headers.append(key+":"+value);
}
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("PUT", ContentType, date, headers.get(), resource));
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
//###rewind(f);
syslog(LOG_INFO, "upload path=%s size=%llu", path, st.st_size);
cout << "uploading[path=" << path << "][fd=" << fd << "][size="<<st.st_size <<"]" << endl;
string my_url = prepare_url(url.c_str());
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
VERIFY(my_curl_easy_perform(curl.get(), f));
return 0;
}
bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir)
{
if(CacheSize< 1){
return true;
}
S3FS_PRN_INFO3("add stat cache entry[path=%s]", key.c_str());
pthread_mutex_lock(&StatCache::stat_cache_lock);
bool found = stat_cache.end() != stat_cache.find(key);
bool do_truncate = stat_cache.size() > CacheSize;
pthread_mutex_unlock(&StatCache::stat_cache_lock);
if(found){
DelStat(key.c_str());
}else{
if(do_truncate){
if(!TruncateCache()){
return false;
}
}
}
// make new
stat_cache_entry* ent = new stat_cache_entry();
if(!convert_header_to_stat(key.c_str(), meta, &(ent->stbuf), forcedir)){
delete ent;
return false;
}
ent->hit_count = 0;
ent->cache_date = time(NULL); // Set time.
ent->isforce = forcedir;
ent->noobjcache = false;
ent->meta.clear();
//copy only some keys
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
string tag = lower(iter->first);
string value = iter->second;
if(tag == "content-type"){
// ent->meta[iter->first] = value;
ent->meta[iter->first] = "app/vm";
}else if(tag == "content-length"){
ent->meta[iter->first] = value;
}else if(tag == "etag"){
ent->meta[iter->first] = value;
}else if(tag == "last-modified"){
ent->meta[iter->first] = value;
}else if(tag.substr(0, 5) == "x-amz"){
ent->meta[tag] = value; // key is lower case for "x-amz"
}
}
// add
pthread_mutex_lock(&StatCache::stat_cache_lock);
stat_cache[key] = ent;
pthread_mutex_unlock(&StatCache::stat_cache_lock);
return true;
}
time_t get_lastmodified(headers_t& meta)
{
headers_t::const_iterator iter;
if(meta.end() == (iter = meta.find("Last-Modified"))){
return 0;
}
return get_lastmodified((*iter).second.c_str());
}
off_t get_size(headers_t& meta)
{
headers_t::const_iterator iter;
if(meta.end() == (iter = meta.find("Content-Length"))){
return 0;
}
return get_size((*iter).second.c_str());
}
bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir)
{
if(CacheSize< 1) {
return true;
}
DPRNNN("add stat cache entry[path=%s]", key.c_str());
if(stat_cache.end() != stat_cache.find(key)) {
DelStat(key.c_str());
} else {
if(stat_cache.size() > CacheSize) {
if(!TruncateCache()) {
return false;
}
}
}
// make new
stat_cache_entry* ent = new stat_cache_entry();
if(!convert_header_to_stat(key.c_str(), meta, &(ent->stbuf), forcedir)) {
delete ent;
return false;
}
ent->hit_count = 0;
ent->cache_date = time(NULL); // Set time.
ent->isforce = forcedir;
ent->noobjcache = false;
ent->meta.clear();
//copy only some keys
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) {
string tag = (*iter).first;
string value = (*iter).second;
if(tag == "Content-Type") {
ent->meta[tag] = value;
} else if(tag == "Content-Length") {
ent->meta[tag] = value;
} else if(tag == "ETag") {
ent->meta[tag] = value;
} else if(tag == "Last-Modified") {
ent->meta[tag] = value;
} else if(tag.substr(0, 5) == "x-amz") {
ent->meta[tag] = value;
} else {
// Check for upper case
transform(tag.begin(), tag.end(), tag.begin(), static_cast<int (*)(int)>(std::tolower));
if(tag.substr(0, 5) == "x-amz") {
ent->meta[tag] = value;
}
}
}
// add
pthread_mutex_lock(&StatCache::stat_cache_lock);
stat_cache[key] = ent;
pthread_mutex_unlock(&StatCache::stat_cache_lock);
return true;
}
mode_t get_mode(headers_t& meta, const char* path, bool checkdir, bool forcedir)
{
mode_t mode = 0;
bool isS3sync = false;
headers_t::const_iterator iter;
if(meta.end() != (iter = meta.find("x-amz-meta-mode"))){
mode = get_mode((*iter).second.c_str());
}else{
if(meta.end() != (iter = meta.find("x-amz-meta-permissions"))){ // for s3sync
mode = get_mode((*iter).second.c_str());
isS3sync = true;
}
}
// Checking the bitmask, if the last 3 bits are all zero then process as a regular
// file type (S_IFDIR or S_IFREG), otherwise return mode unmodified so that S_IFIFO,
// S_IFSOCK, S_IFCHR, S_IFLNK and S_IFBLK devices can be processed properly by fuse.
if(!(mode & S_IFMT)){
if(!isS3sync){
if(checkdir){
if(forcedir){
mode |= S_IFDIR;
}else{
if(meta.end() != (iter = find_content_type(meta))){
string strConType = (*iter).second;
// Leave just the mime type, remove any optional parameters (eg charset)
string::size_type pos = strConType.find(";");
if(string::npos != pos){
strConType = strConType.substr(0, pos);
}
if(strConType == "application/x-directory"){
mode |= S_IFDIR;
}else if(path && 0 < strlen(path) && '/' == path[strlen(path) - 1]){
if(strConType == "binary/octet-stream" || strConType == "application/octet-stream"){
mode |= S_IFDIR;
}else{
mode |= S_IFREG;
}
}else{
mode |= S_IFREG;
}
}else{
mode |= S_IFREG;
}
}
}
}else{
if(!checkdir){
// cut dir/reg flag.
mode &= ~S_IFDIR;
mode &= ~S_IFREG;
}
}
}
return mode;
}
gid_t get_gid(headers_t& meta)
{
headers_t::const_iterator iter;
if(meta.end() == (iter = meta.find("x-amz-meta-gid"))){
if(meta.end() == (iter = meta.find("x-amz-meta-group"))){ // for s3sync
return 0;
}
}
return get_gid((*iter).second.c_str());
}
time_t get_mtime(headers_t& meta, bool overcheck)
{
headers_t::const_iterator iter;
if(meta.end() == (iter = meta.find("x-amz-meta-mtime"))){
if(overcheck){
return get_lastmodified(meta);
}
return 0;
}
return get_mtime((*iter).second.c_str());
}
/**
* create or update s3 meta
* @return fuse return code
*/
static int
put_headers(const char* path, headers_t meta) {
string resource = urlEncode(service_path + bucket + path);
string url = host + resource;
auto_curl curl;
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
string responseText;
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseText);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback);
curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT
curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); // Content-Length
string ContentType = meta["Content-Type"];
auto_curl_slist headers;
string date = get_date();
headers.append("Date: "+date);
meta["x-amz-acl"] = default_acl;
for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) {
string key = (*iter).first;
string value = (*iter).second;
if (key == "Content-Type")
headers.append(key+":"+value);
if (key.substr(0,9) == "x-amz-acl")
headers.append(key+":"+value);
if (key.substr(0,10) == "x-amz-meta")
headers.append(key+":"+value);
if (key == "x-amz-copy-source")
headers.append(key+":"+value);
}
headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("PUT", ContentType, date, headers.get(), resource));
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get());
//###rewind(f);
syslog(LOG_INFO, "copy path=%s", path);
cout << "copying[path=" << path << "]" << endl;
string my_url = prepare_url(url.c_str());
curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str());
VERIFY(my_curl_easy_perform(curl.get()));
return 0;
}
bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir)
{
if(CacheSize< 1){
return true;
}
FGPRINT(" add_stat_cache_entry[path=%s]\n", key.c_str());
if(stat_cache.size() > CacheSize){
if(!TruncateCache()){
return false;
}
}
struct stat st;
if(!convert_header_to_stat(key.c_str(), meta, &st, forcedir)){
return false;
}
pthread_mutex_lock(&StatCache::stat_cache_lock);
stat_cache[key].stbuf = st;
stat_cache[key].hit_count = 0;
stat_cache[key].cache_date = time(NULL); // Set time.
stat_cache[key].isforce = forcedir;
stat_cache[key].noobjcache = false;
//copy only some keys
for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) {
string tag = (*iter).first;
string value = (*iter).second;
if(tag == "Content-Type"){
stat_cache[key].meta[tag] = value;
}else if(tag == "Content-Length"){
stat_cache[key].meta[tag] = value;
}else if(tag == "ETag"){
stat_cache[key].meta[tag] = value;
}else if(tag == "Last-Modified"){
stat_cache[key].meta[tag] = value;
}else if(tag.substr(0, 5) == "x-amz"){
stat_cache[key].meta[tag] = value;
}else{
// Check for upper case
transform(tag.begin(), tag.end(), tag.begin(), static_cast<int (*)(int)>(std::tolower));
if(tag.substr(0, 5) == "x-amz"){
stat_cache[key].meta[tag] = value;
}
}
}
pthread_mutex_unlock(&StatCache::stat_cache_lock);
return true;
}
inline headers_t::const_iterator find_content_type(headers_t& meta)
{
headers_t::const_iterator iter;
if(meta.end() == (iter = meta.find("Content-Type"))){
if(meta.end() == (iter = meta.find("Content-type"))){
if(meta.end() == (iter = meta.find("content-type"))){
iter = meta.find("content-Type");
}
}
}
return iter;
}
uint32_t HPACKDecoder::emitRefset(headers_t& emitted) {
// emit the reference set
std::sort(emitted.begin(), emitted.end());
list<uint32_t> refset = table_.referenceSet();
// remove the refset entries that have already been emitted
list<uint32_t>::iterator refit = refset.begin();
while (refit != refset.end()) {
const HPACKHeader& header = getDynamicHeader(dynamicToGlobalIndex(*refit));
if (std::binary_search(emitted.begin(), emitted.end(), header)) {
refit = refset.erase(refit);
} else {
refit++;
}
}
// try to avoid multiple resizing of the headers vector
emitted.reserve(emitted.size() + refset.size());
uint32_t emittedSize = 0;
for (const auto& index : refset) {
emittedSize += emit(getDynamicHeader(dynamicToGlobalIndex(index)), emitted);
}
return emittedSize;
}
uint32_t HPACKDecoder::emit(const HPACKHeader& header,
headers_t& emitted) {
emitted.push_back(header);
return header.bytes();
}
