int getFileList(const string& path, vector<string>& fl, Sector& client, unsigned int thresh) { SNode attr; if (client.stat(path.c_str(), attr) < 0) return -1; if (attr.m_bIsDir) { vector<SNode> subdir; client.list(path, subdir); for (vector<SNode>::iterator i = subdir.begin(); i != subdir.end(); ++ i) { if (i->m_bIsDir) getFileList(path + "/" + i->m_strName, fl, client, thresh); else if ((i->m_sLocation.size() < thresh) && (int(i->m_sLocation.size()) < i->m_iReplicaNum)) fl.push_back(path + "/" + i->m_strName); } } else if ((attr.m_sLocation.size() < thresh) && (int(attr.m_sLocation.size()) < attr.m_iReplicaNum)) { fl.push_back(path); } return fl.size(); }
int getFileList(const string& path, vector<string>& fl, Sector& client) { SNode attr; if (client.stat(path, attr) < 0) return -1; fl.push_back(path); if (attr.m_bIsDir) { vector<SNode> subdir; client.list(path, subdir); for (vector<SNode>::iterator i = subdir.begin(); i != subdir.end(); ++ i) { if (i->m_bIsDir) getFileList(path + "/" + i->m_strName, fl, client); else fl.push_back(path + "/" + i->m_strName); } } return fl.size(); }
int main(int argc, char** argv) { if (argc < 2) { help(); return -1; } unsigned int thresh = 65536; int timeout = 3600 * 24; // default wait for one day CmdLineParser clp; if (clp.parse(argc, argv) < 0) { help(); return -1; } for (map<string, string>::const_iterator i = clp.m_mDFlags.begin(); i != clp.m_mDFlags.end(); ++ i) { if (i->first == "w") thresh = atoi(i->second.c_str()); else if (i->first == "t") { timeout = atoi(i->second.c_str()); if (timeout < 0) timeout = INT_MAX; } else { help(); return -1; } } Sector client; if (Utility::login(client) < 0) return -1; list<string> filelist; for (vector<string>::iterator i = clp.m_vParams.begin(); i < clp.m_vParams.end(); ++ i) { vector<string> fl; fl.clear(); bool wc = WildCard::isWildCard(*i); if (!wc) { SNode attr; if (client.stat(*i, attr) < 0) { cerr << "ERROR: source file does not exist.\n"; return -1; } getFileList(*i, fl, client, thresh); } else { string path = *i; string orig = path; size_t p = path.rfind('/'); if (p == string::npos) path = "/"; else { path = path.substr(0, p); orig = orig.substr(p + 1, orig.length() - p); } vector<SNode> filelist; int r = client.list(path, filelist); if (r < 0) cerr << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl; for (vector<SNode>::iterator i = filelist.begin(); i != filelist.end(); ++ i) { if (WildCard::match(orig, i->m_strName)) getFileList(path + "/" + i->m_strName, fl, client, thresh); } } filelist.insert(filelist.end(), fl.begin(), fl.end()); } int result = -1; timeval t; gettimeofday(&t, NULL); int exp_time = t.tv_sec + timeout; int interval = 1; int total = -1; int new_total = 0; while (!filelist.empty()) { new_total = 0; for (list<string>::iterator i = filelist.begin(); i != filelist.end();) { SNode sn; if (client.stat(*i, sn) < 0) { cout << "file " << sn.m_strName << " is lost.\n"; break; } int diff = (sn.m_iReplicaNum > int(thresh)) ? (thresh - sn.m_sLocation.size()) : (sn.m_iReplicaNum - int(sn.m_sLocation.size())); if (diff <= 0) { list<string>::iterator j = i; ++ i; filelist.erase(j); } else { if ((result = client.copy(*i, *i)) < 0) { Utility::print_error(result); list<string>::iterator j = i; ++ i; filelist.erase(j); break; } ++ i; new_total += diff; } } timeval curr_time; gettimeofday(&curr_time, NULL); if (curr_time.tv_sec > exp_time) { cout << "timeout.\n"; break; } //TODO: sleep diff for the last time if (interval < exp_time - curr_time.tv_sec) sleep(interval); else sleep(exp_time - curr_time.tv_sec); // wait longer if no progress, otherwise continue to wait for 1 sec if ((total > 0) && (new_total < total)) { total = new_total; interval = 1; } else if (interval < 16) { interval <<= 1; } } if (filelist.empty()) { cout << "all files have enough replicas.\n"; result = 0; } Utility::logout(client); return result; }
int main(int argc, char** argv) { logger::config( "/tmp", "sector-download" ); for( int arg = 0; arg < argc; ++arg ) log().debug << argv[ arg ] << ' '; log().debug << std::endl; if (argc < 3) { help(argv[0]); log().error << "Invalid command-line syntax, exiting with rc = -1" << std::endl; return -1; } CmdLineParser clp; if (clp.parse(argc, argv) < 0) { help(argv[0]); log().error << "Invalid command-line syntax, exiting with rc = -1" << std::endl; return -1; } if (clp.m_vParams.size() < 2) { help(argv[0]); log().error << "Invalid command-line syntax, exiting with rc = -1" << std::endl; return -1; } bool encryption = false; bool resume = false; for (vector<string>::const_iterator i = clp.m_vSFlags.begin(); i != clp.m_vSFlags.end(); ++ i) { if (*i == "e") encryption = true; else if( *i == "smart" ) resume = true; else { help(argv[0]); log().error << "Invalid command-line syntax, exiting with rc = -1" << std::endl; return -1; } } string newdir = *clp.m_vParams.rbegin(); clp.m_vParams.erase(clp.m_vParams.begin() + clp.m_vParams.size() - 1); // check destination directory, which must exist SNode s; int r = LocalFS::stat(newdir, s); if ((r < 0) || !s.m_bIsDir) { cerr << "ERROR: destination directory does not exist.\n"; log().error << "stat failed on destination directory, err = " << r << ", exiting with rc = -1" << std::endl; return -1; } // login to SectorFS Sector client; int rc = Utility::login(client); if (rc < 0) { cerr << "ERROR: failed to log in to sector\n"; log().error << "Client login to master failed with err = " << rc << ", exiting with rc = -1" << std::endl; return -1; } // start downloading all files for (vector<string>::iterator i = clp.m_vParams.begin(); i != clp.m_vParams.end(); ++ i) { vector<string> fl; bool wc = WildCard::isWildCard(*i); if (!wc) { SNode attr; if ((rc = client.stat(*i, attr)) < 0) { cerr << "ERROR: source file does not exist.\n"; log().error << "Failed to stat sector file " << *i << ", err = " << rc << std::endl; return -1; } getFileList(*i, fl, client); } else { string path = *i; string orig = path; size_t p = path.rfind('/'); if (p == string::npos) path = "/"; else { path = path.substr(0, p); orig = orig.substr(p + 1, orig.length() - p); } vector<SNode> filelist; int r = client.list(path, filelist); if (r < 0) cerr << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl; for (vector<SNode>::iterator i = filelist.begin(); i != filelist.end(); ++ i) { if (WildCard::match(orig, i->m_strName)) getFileList(path + "/" + i->m_strName, fl, client); } } string olddir; for (int j = i->length() - 1; j >= 0; -- j) { if (i->c_str()[j] != '/') { olddir = i->substr(0, j); break; } } size_t p = olddir.rfind('/'); if (p == string::npos) olddir = ""; else olddir = olddir.substr(0, p); for (vector<string>::iterator i = fl.begin(); i != fl.end(); ++ i) { string dst = *i; if (olddir.length() > 0) dst.replace(0, olddir.length(), newdir); else dst = newdir + "/" + dst; string localdir = dst.substr(0, dst.rfind('/')); // if localdir does not exist, create it if (LocalFS::stat(localdir, s) < 0) { for (unsigned int p = 1; p < localdir.length(); ++ p) { if (localdir.c_str()[p] == '/') { string substr = localdir.substr(0, p); if ((-1 == ::mkdir(substr.c_str(), S_IRWXU)) && (errno != EEXIST)) { int errno_save = errno; cerr << "ERROR: unable to create local directory " << substr << endl; log().error << "Failed to create local directory " << substr << ", errno = " << errno_save << std::endl; return -1; } } } if ((-1 == ::mkdir(localdir.c_str(), S_IRWXU)) && (errno != EEXIST)) { int errno_save = errno; cerr << "ERROR: unable to create local directory " << localdir << endl; log().error << "Failed to create local directory " << localdir << ", errno = " << errno_save << std::endl; break; } } if (!resume ) { string fileName = *i; if( fileName.rfind( '/' ) != fileName.npos ) fileName = fileName.substr( fileName.rfind( '/' ) + 1 ); string destFile = localdir + '/' + fileName; if( LocalFS::stat( destFile, s ) >= 0 ) { cout << "Destination directory already contains file '" << fileName << "', removing." << endl; log().debug << "Destination directory already contains file " << fileName << " and smart not specified, removing file" << std::endl; if( LocalFS::erase( destFile ) < 0 ) { int save_errno = errno; cerr << "ERROR: could not remove destination file: " << strerror( save_errno ) << endl << "NOT downloading file!" << endl; log().error << "Failed to remove destination file " << fileName << ", aborting this file" << std::endl; return -1; } } } else { string fileName = *i; if( fileName.rfind( '/' ) != fileName.npos ) fileName = fileName.substr( fileName.rfind( '/' ) + 1 ); string destFile = localdir + '/' + fileName; if( LocalFS::stat( destFile, s ) >= 0 ) { cout << "Destination directory already contains file '" << fileName << "', resuming partial download." << endl; log().debug << "Destination file already exists with size " << s.m_llSize << ", resuming partial download" << std::endl; } } if (download(i->c_str(), localdir.c_str(), client, encryption) < 0) { // calculate total available disk size int64_t availdisk = 0; LocalFS::get_dir_space(localdir, availdisk); if (availdisk <= 0) { // if no disk space svailable, no need to try any more cerr << "insufficient local disk space. quit.\n"; log().error << "Insufficient disk space" << std::endl; Utility::logout(client); } return -1; } } } Utility::logout(client); log().debug << "Download completed successfully" << std::endl; return 0; }
int main(int argc, char** argv) { if (argc != 2) { cout << "USAGE: rm <dir>\n"; return -1; } Sector client; Session s; s.loadInfo("../conf/client.conf"); if (client.init(s.m_ClientConf.m_strMasterIP, s.m_ClientConf.m_iMasterPort) < 0) return -1; if (client.login(s.m_ClientConf.m_strUserName, s.m_ClientConf.m_strPassword, s.m_ClientConf.m_strCertificate.c_str()) < 0) return -1; string path = argv[1]; bool wc = WildCard::isWildCard(path); if (!wc) { int r = client.remove(path); if (r == SectorError::E_NOEMPTY) { if (isRecursive(path)) client.rmr(path); } else if (r < 0) cout << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl; } else { string orig = path; size_t p = path.rfind('/'); if (p == string::npos) path = "/"; else { path = path.substr(0, p); orig = orig.substr(p + 1, orig.length() - p); } vector<SNode> filelist; int r = client.list(path, filelist); if (r < 0) cout << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl; bool recursive = false; vector<string> filtered; for (vector<SNode>::iterator i = filelist.begin(); i != filelist.end(); ++ i) { if (WildCard::match(orig, i->m_strName)) { if (recursive) client.rmr(path + "/" + i->m_strName); else { r = client.remove(path + "/" + i->m_strName); if (r == SectorError::E_NOEMPTY) { recursive = isRecursive(path + "/" + i->m_strName); if (recursive) client.rmr(path + "/" + i->m_strName); } else if (r < 0) cout << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl; } } } } client.logout(); client.close(); return 1; }