// // Returns it whether it is an object with need checking in detail. // If this function returns true, the object is possible to be directory // and is needed checking detail(searching sub object). // bool is_need_check_obj_detail(headers_t& meta) { headers_t::const_iterator iter; // directory object is Content-Length as 0. if(0 != get_size(meta)){ return false; } // if the object has x-amz-meta information, checking is no more. if(meta.end() != meta.find("x-amz-meta-mode") || meta.end() != meta.find("x-amz-meta-mtime") || meta.end() != meta.find("x-amz-meta-uid") || meta.end() != meta.find("x-amz-meta-gid") || meta.end() != meta.find("x-amz-meta-owner") || meta.end() != meta.find("x-amz-meta-group") || meta.end() != meta.find("x-amz-meta-permissions") ) { return false; } // if there is not Content-Type, or Content-Type is "x-directory", // checking is no more. if(meta.end() == (iter = find_content_type(meta))){ return false; } if("application/x-directory" == (*iter).second){ return false; } return true; }
mode_t get_mode(headers_t& meta, const char* path, bool checkdir, bool forcedir) { mode_t mode = 0; bool isS3sync = false; headers_t::const_iterator iter; if(meta.end() != (iter = meta.find("x-amz-meta-mode"))){ mode = get_mode((*iter).second.c_str()); }else{ if(meta.end() != (iter = meta.find("x-amz-meta-permissions"))){ // for s3sync mode = get_mode((*iter).second.c_str()); isS3sync = true; } } // Checking the bitmask, if the last 3 bits are all zero then process as a regular // file type (S_IFDIR or S_IFREG), otherwise return mode unmodified so that S_IFIFO, // S_IFSOCK, S_IFCHR, S_IFLNK and S_IFBLK devices can be processed properly by fuse. if(!(mode & S_IFMT)){ if(!isS3sync){ if(checkdir){ if(forcedir){ mode |= S_IFDIR; }else{ if(meta.end() != (iter = find_content_type(meta))){ string strConType = (*iter).second; // Leave just the mime type, remove any optional parameters (eg charset) string::size_type pos = strConType.find(";"); if(string::npos != pos){ strConType = strConType.substr(0, pos); } if(strConType == "application/x-directory"){ mode |= S_IFDIR; }else if(path && 0 < strlen(path) && '/' == path[strlen(path) - 1]){ if(strConType == "binary/octet-stream" || strConType == "application/octet-stream"){ mode |= S_IFDIR; }else{ mode |= S_IFREG; } }else{ mode |= S_IFREG; } }else{ mode |= S_IFREG; } } } }else{ if(!checkdir){ // cut dir/reg flag. mode &= ~S_IFDIR; mode &= ~S_IFREG; } } } return mode; }
gid_t get_gid(headers_t& meta) { headers_t::const_iterator iter; if(meta.end() == (iter = meta.find("x-amz-meta-gid"))){ if(meta.end() == (iter = meta.find("x-amz-meta-group"))){ // for s3sync return 0; } } return get_gid((*iter).second.c_str()); }
inline headers_t::const_iterator find_content_type(headers_t& meta) { headers_t::const_iterator iter; if(meta.end() == (iter = meta.find("Content-Type"))){ if(meta.end() == (iter = meta.find("Content-type"))){ if(meta.end() == (iter = meta.find("content-type"))){ iter = meta.find("content-Type"); } } } return iter; }
table_t(headers_t headers, formatters_t formatters, alignments_t alignments, size_t sort_index = 0, sort_direction direction = sort_direction::ascending) : formatters(formatters), alignments(alignments), sort_index(sort_index), direction(direction) { std::transform(headers.begin(), headers.end(), this->headers.begin(), [](auto& x) { return to_upper(x); }); update_column_max_width(headers); }
/** * create or update s3 object * @return fuse return code */ static int put_local_fd(const char* path, headers_t meta, int fd) { string resource = urlEncode(service_path + bucket + path); string url = host + resource; struct stat st; if (fstat(fd, &st) == -1) Yikes(-errno); auto_curl curl; curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); string responseText; curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseText); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(st.st_size)); // Content-Length FILE* f = fdopen(fd, "rb"); if (f == 0) Yikes(-errno); curl_easy_setopt(curl, CURLOPT_INFILE, f); string ContentType = meta["Content-Type"]; auto_curl_slist headers; string date = get_date(); headers.append("Date: "+date); meta["x-amz-acl"] = default_acl; for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) { string key = (*iter).first; string value = (*iter).second; if (key == "Content-Type") headers.append(key+":"+value); if (key.substr(0,9) == "x-amz-acl") headers.append(key+":"+value); if (key.substr(0,10) == "x-amz-meta") headers.append(key+":"+value); } headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("PUT", ContentType, date, headers.get(), resource)); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); //###rewind(f); syslog(LOG_INFO, "upload path=%s size=%llu", path, st.st_size); cout << "uploading[path=" << path << "][fd=" << fd << "][size="<<st.st_size <<"]" << endl; string my_url = prepare_url(url.c_str()); curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str()); VERIFY(my_curl_easy_perform(curl.get(), f)); return 0; }
bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir) { if(CacheSize< 1){ return true; } S3FS_PRN_INFO3("add stat cache entry[path=%s]", key.c_str()); pthread_mutex_lock(&StatCache::stat_cache_lock); bool found = stat_cache.end() != stat_cache.find(key); bool do_truncate = stat_cache.size() > CacheSize; pthread_mutex_unlock(&StatCache::stat_cache_lock); if(found){ DelStat(key.c_str()); }else{ if(do_truncate){ if(!TruncateCache()){ return false; } } } // make new stat_cache_entry* ent = new stat_cache_entry(); if(!convert_header_to_stat(key.c_str(), meta, &(ent->stbuf), forcedir)){ delete ent; return false; } ent->hit_count = 0; ent->cache_date = time(NULL); // Set time. ent->isforce = forcedir; ent->noobjcache = false; ent->meta.clear(); //copy only some keys for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ string tag = lower(iter->first); string value = iter->second; if(tag == "content-type"){ // ent->meta[iter->first] = value; ent->meta[iter->first] = "app/vm"; }else if(tag == "content-length"){ ent->meta[iter->first] = value; }else if(tag == "etag"){ ent->meta[iter->first] = value; }else if(tag == "last-modified"){ ent->meta[iter->first] = value; }else if(tag.substr(0, 5) == "x-amz"){ ent->meta[tag] = value; // key is lower case for "x-amz" } } // add pthread_mutex_lock(&StatCache::stat_cache_lock); stat_cache[key] = ent; pthread_mutex_unlock(&StatCache::stat_cache_lock); return true; }
time_t get_lastmodified(headers_t& meta) { headers_t::const_iterator iter; if(meta.end() == (iter = meta.find("Last-Modified"))){ return 0; } return get_lastmodified((*iter).second.c_str()); }
off_t get_size(headers_t& meta) { headers_t::const_iterator iter; if(meta.end() == (iter = meta.find("Content-Length"))){ return 0; } return get_size((*iter).second.c_str()); }
bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir) { if(CacheSize< 1) { return true; } DPRNNN("add stat cache entry[path=%s]", key.c_str()); if(stat_cache.end() != stat_cache.find(key)) { DelStat(key.c_str()); } else { if(stat_cache.size() > CacheSize) { if(!TruncateCache()) { return false; } } } // make new stat_cache_entry* ent = new stat_cache_entry(); if(!convert_header_to_stat(key.c_str(), meta, &(ent->stbuf), forcedir)) { delete ent; return false; } ent->hit_count = 0; ent->cache_date = time(NULL); // Set time. ent->isforce = forcedir; ent->noobjcache = false; ent->meta.clear(); //copy only some keys for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) { string tag = (*iter).first; string value = (*iter).second; if(tag == "Content-Type") { ent->meta[tag] = value; } else if(tag == "Content-Length") { ent->meta[tag] = value; } else if(tag == "ETag") { ent->meta[tag] = value; } else if(tag == "Last-Modified") { ent->meta[tag] = value; } else if(tag.substr(0, 5) == "x-amz") { ent->meta[tag] = value; } else { // Check for upper case transform(tag.begin(), tag.end(), tag.begin(), static_cast<int (*)(int)>(std::tolower)); if(tag.substr(0, 5) == "x-amz") { ent->meta[tag] = value; } } } // add pthread_mutex_lock(&StatCache::stat_cache_lock); stat_cache[key] = ent; pthread_mutex_unlock(&StatCache::stat_cache_lock); return true; }
time_t get_mtime(headers_t& meta, bool overcheck) { headers_t::const_iterator iter; if(meta.end() == (iter = meta.find("x-amz-meta-mtime"))){ if(overcheck){ return get_lastmodified(meta); } return 0; } return get_mtime((*iter).second.c_str()); }
/** * create or update s3 meta * @return fuse return code */ static int put_headers(const char* path, headers_t meta) { string resource = urlEncode(service_path + bucket + path); string url = host + resource; auto_curl curl; curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); string responseText; curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseText); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); curl_easy_setopt(curl, CURLOPT_UPLOAD, true); // HTTP PUT curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); // Content-Length string ContentType = meta["Content-Type"]; auto_curl_slist headers; string date = get_date(); headers.append("Date: "+date); meta["x-amz-acl"] = default_acl; for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) { string key = (*iter).first; string value = (*iter).second; if (key == "Content-Type") headers.append(key+":"+value); if (key.substr(0,9) == "x-amz-acl") headers.append(key+":"+value); if (key.substr(0,10) == "x-amz-meta") headers.append(key+":"+value); if (key == "x-amz-copy-source") headers.append(key+":"+value); } headers.append("Authorization: AWS "+AWSAccessKeyId+":"+calc_signature("PUT", ContentType, date, headers.get(), resource)); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers.get()); //###rewind(f); syslog(LOG_INFO, "copy path=%s", path); cout << "copying[path=" << path << "]" << endl; string my_url = prepare_url(url.c_str()); curl_easy_setopt(curl, CURLOPT_URL, my_url.c_str()); VERIFY(my_curl_easy_perform(curl.get())); return 0; }
uint32_t HPACKDecoder::emitRefset(headers_t& emitted) { // emit the reference set std::sort(emitted.begin(), emitted.end()); list<uint32_t> refset = table_.referenceSet(); // remove the refset entries that have already been emitted list<uint32_t>::iterator refit = refset.begin(); while (refit != refset.end()) { const HPACKHeader& header = getDynamicHeader(dynamicToGlobalIndex(*refit)); if (std::binary_search(emitted.begin(), emitted.end(), header)) { refit = refset.erase(refit); } else { refit++; } } // try to avoid multiple resizing of the headers vector emitted.reserve(emitted.size() + refset.size()); uint32_t emittedSize = 0; for (const auto& index : refset) { emittedSize += emit(getDynamicHeader(dynamicToGlobalIndex(index)), emitted); } return emittedSize; }
bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir) { if(CacheSize< 1){ return true; } FGPRINT(" add_stat_cache_entry[path=%s]\n", key.c_str()); if(stat_cache.size() > CacheSize){ if(!TruncateCache()){ return false; } } struct stat st; if(!convert_header_to_stat(key.c_str(), meta, &st, forcedir)){ return false; } pthread_mutex_lock(&StatCache::stat_cache_lock); stat_cache[key].stbuf = st; stat_cache[key].hit_count = 0; stat_cache[key].cache_date = time(NULL); // Set time. stat_cache[key].isforce = forcedir; stat_cache[key].noobjcache = false; //copy only some keys for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) { string tag = (*iter).first; string value = (*iter).second; if(tag == "Content-Type"){ stat_cache[key].meta[tag] = value; }else if(tag == "Content-Length"){ stat_cache[key].meta[tag] = value; }else if(tag == "ETag"){ stat_cache[key].meta[tag] = value; }else if(tag == "Last-Modified"){ stat_cache[key].meta[tag] = value; }else if(tag.substr(0, 5) == "x-amz"){ stat_cache[key].meta[tag] = value; }else{ // Check for upper case transform(tag.begin(), tag.end(), tag.begin(), static_cast<int (*)(int)>(std::tolower)); if(tag.substr(0, 5) == "x-amz"){ stat_cache[key].meta[tag] = value; } } } pthread_mutex_unlock(&StatCache::stat_cache_lock); return true; }