int getFileList(const string& path, vector<string>& fl, Sector& client, unsigned int thresh)
{
SNode attr;
if (client.stat(path.c_str(), attr) < 0)
return -1;
if (attr.m_bIsDir)
{
vector<SNode> subdir;
client.list(path, subdir);
for (vector<SNode>::iterator i = subdir.begin(); i != subdir.end(); ++ i)
{
if (i->m_bIsDir)
getFileList(path + "/" + i->m_strName, fl, client, thresh);
else if ((i->m_sLocation.size() < thresh) && (int(i->m_sLocation.size()) < i->m_iReplicaNum))
fl.push_back(path + "/" + i->m_strName);
}
}
else if ((attr.m_sLocation.size() < thresh) && (int(attr.m_sLocation.size()) < attr.m_iReplicaNum))
{
fl.push_back(path);
}
return fl.size();
}
int getFileList(const string& path, vector<string>& fl, Sector& client)
{
SNode attr;
if (client.stat(path, attr) < 0)
return -1;
fl.push_back(path);
if (attr.m_bIsDir)
{
vector<SNode> subdir;
client.list(path, subdir);
for (vector<SNode>::iterator i = subdir.begin(); i != subdir.end(); ++ i)
{
if (i->m_bIsDir)
getFileList(path + "/" + i->m_strName, fl, client);
else
fl.push_back(path + "/" + i->m_strName);
}
}
return fl.size();
}
int main(int argc, char** argv)
{
if (argc < 2)
{
help();
return -1;
}
unsigned int thresh = 65536;
int timeout = 3600 * 24; // default wait for one day
CmdLineParser clp;
if (clp.parse(argc, argv) < 0)
{
help();
return -1;
}
for (map<string, string>::const_iterator i = clp.m_mDFlags.begin(); i != clp.m_mDFlags.end(); ++ i)
{
if (i->first == "w")
thresh = atoi(i->second.c_str());
else if (i->first == "t")
{
timeout = atoi(i->second.c_str());
if (timeout < 0)
timeout = INT_MAX;
}
else
{
help();
return -1;
}
}
Sector client;
if (Utility::login(client) < 0)
return -1;
list<string> filelist;
for (vector<string>::iterator i = clp.m_vParams.begin(); i < clp.m_vParams.end(); ++ i)
{
vector<string> fl;
fl.clear();
bool wc = WildCard::isWildCard(*i);
if (!wc)
{
SNode attr;
if (client.stat(*i, attr) < 0)
{
cerr << "ERROR: source file does not exist.\n";
return -1;
}
getFileList(*i, fl, client, thresh);
}
else
{
string path = *i;
string orig = path;
size_t p = path.rfind('/');
if (p == string::npos)
path = "/";
else
{
path = path.substr(0, p);
orig = orig.substr(p + 1, orig.length() - p);
}
vector<SNode> filelist;
int r = client.list(path, filelist);
if (r < 0)
cerr << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl;
for (vector<SNode>::iterator i = filelist.begin(); i != filelist.end(); ++ i)
{
if (WildCard::match(orig, i->m_strName))
getFileList(path + "/" + i->m_strName, fl, client, thresh);
}
}
filelist.insert(filelist.end(), fl.begin(), fl.end());
}
int result = -1;
timeval t;
gettimeofday(&t, NULL);
int exp_time = t.tv_sec + timeout;
int interval = 1;
int total = -1;
int new_total = 0;
while (!filelist.empty())
{
new_total = 0;
for (list<string>::iterator i = filelist.begin(); i != filelist.end();)
{
SNode sn;
if (client.stat(*i, sn) < 0)
{
cout << "file " << sn.m_strName << " is lost.\n";
break;
}
int diff = (sn.m_iReplicaNum > int(thresh)) ? (thresh - sn.m_sLocation.size()) : (sn.m_iReplicaNum - int(sn.m_sLocation.size()));
if (diff <= 0)
{
list<string>::iterator j = i;
++ i;
filelist.erase(j);
}
else
{
if ((result = client.copy(*i, *i)) < 0)
{
Utility::print_error(result);
list<string>::iterator j = i;
++ i;
filelist.erase(j);
break;
}
++ i;
new_total += diff;
}
}
timeval curr_time;
gettimeofday(&curr_time, NULL);
if (curr_time.tv_sec > exp_time)
{
cout << "timeout.\n";
break;
}
//TODO: sleep diff for the last time
if (interval < exp_time - curr_time.tv_sec)
sleep(interval);
else
sleep(exp_time - curr_time.tv_sec);
// wait longer if no progress, otherwise continue to wait for 1 sec
if ((total > 0) && (new_total < total))
{
total = new_total;
interval = 1;
}
else if (interval < 16)
{
interval <<= 1;
}
}
if (filelist.empty())
{
cout << "all files have enough replicas.\n";
result = 0;
}
Utility::logout(client);
return result;
}
int main(int argc, char** argv)
{
logger::config( "/tmp", "sector-download" );
for( int arg = 0; arg < argc; ++arg )
log().debug << argv[ arg ] << ' ';
log().debug << std::endl;
if (argc < 3)
{
help(argv[0]);
log().error << "Invalid command-line syntax, exiting with rc = -1" << std::endl;
return -1;
}
CmdLineParser clp;
if (clp.parse(argc, argv) < 0)
{
help(argv[0]);
log().error << "Invalid command-line syntax, exiting with rc = -1" << std::endl;
return -1;
}
if (clp.m_vParams.size() < 2)
{
help(argv[0]);
log().error << "Invalid command-line syntax, exiting with rc = -1" << std::endl;
return -1;
}
bool encryption = false;
bool resume = false;
for (vector<string>::const_iterator i = clp.m_vSFlags.begin(); i != clp.m_vSFlags.end(); ++ i)
{
if (*i == "e")
encryption = true;
else if( *i == "smart" )
resume = true;
else
{
help(argv[0]);
log().error << "Invalid command-line syntax, exiting with rc = -1" << std::endl;
return -1;
}
}
string newdir = *clp.m_vParams.rbegin();
clp.m_vParams.erase(clp.m_vParams.begin() + clp.m_vParams.size() - 1);
// check destination directory, which must exist
SNode s;
int r = LocalFS::stat(newdir, s);
if ((r < 0) || !s.m_bIsDir)
{
cerr << "ERROR: destination directory does not exist.\n";
log().error << "stat failed on destination directory, err = " << r << ", exiting with rc = -1" << std::endl;
return -1;
}
// login to SectorFS
Sector client;
int rc = Utility::login(client);
if (rc < 0)
{
cerr << "ERROR: failed to log in to sector\n";
log().error << "Client login to master failed with err = " << rc << ", exiting with rc = -1" << std::endl;
return -1;
}
// start downloading all files
for (vector<string>::iterator i = clp.m_vParams.begin(); i != clp.m_vParams.end(); ++ i)
{
vector<string> fl;
bool wc = WildCard::isWildCard(*i);
if (!wc)
{
SNode attr;
if ((rc = client.stat(*i, attr)) < 0)
{
cerr << "ERROR: source file does not exist.\n";
log().error << "Failed to stat sector file " << *i << ", err = " << rc << std::endl;
return -1;
}
getFileList(*i, fl, client);
}
else
{
string path = *i;
string orig = path;
size_t p = path.rfind('/');
if (p == string::npos)
path = "/";
else
{
path = path.substr(0, p);
orig = orig.substr(p + 1, orig.length() - p);
}
vector<SNode> filelist;
int r = client.list(path, filelist);
if (r < 0)
cerr << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl;
for (vector<SNode>::iterator i = filelist.begin(); i != filelist.end(); ++ i)
{
if (WildCard::match(orig, i->m_strName))
getFileList(path + "/" + i->m_strName, fl, client);
}
}
string olddir;
for (int j = i->length() - 1; j >= 0; -- j)
{
if (i->c_str()[j] != '/')
{
olddir = i->substr(0, j);
break;
}
}
size_t p = olddir.rfind('/');
if (p == string::npos)
olddir = "";
else
olddir = olddir.substr(0, p);
for (vector<string>::iterator i = fl.begin(); i != fl.end(); ++ i)
{
string dst = *i;
if (olddir.length() > 0)
dst.replace(0, olddir.length(), newdir);
else
dst = newdir + "/" + dst;
string localdir = dst.substr(0, dst.rfind('/'));
// if localdir does not exist, create it
if (LocalFS::stat(localdir, s) < 0)
{
for (unsigned int p = 1; p < localdir.length(); ++ p)
{
if (localdir.c_str()[p] == '/')
{
string substr = localdir.substr(0, p);
if ((-1 == ::mkdir(substr.c_str(), S_IRWXU)) && (errno != EEXIST))
{
int errno_save = errno;
cerr << "ERROR: unable to create local directory " << substr << endl;
log().error << "Failed to create local directory " << substr << ", errno = " << errno_save << std::endl;
return -1;
}
}
}
if ((-1 == ::mkdir(localdir.c_str(), S_IRWXU)) && (errno != EEXIST))
{
int errno_save = errno;
cerr << "ERROR: unable to create local directory " << localdir << endl;
log().error << "Failed to create local directory " << localdir << ", errno = " << errno_save << std::endl;
break;
}
}
if (!resume )
{
string fileName = *i;
if( fileName.rfind( '/' ) != fileName.npos )
fileName = fileName.substr( fileName.rfind( '/' ) + 1 );
string destFile = localdir + '/' + fileName;
if( LocalFS::stat( destFile, s ) >= 0 )
{
cout << "Destination directory already contains file '" << fileName << "', removing." << endl;
log().debug << "Destination directory already contains file " << fileName << " and smart not specified, removing file" << std::endl;
if( LocalFS::erase( destFile ) < 0 )
{
int save_errno = errno;
cerr << "ERROR: could not remove destination file: " << strerror( save_errno ) << endl
<< "NOT downloading file!" << endl;
log().error << "Failed to remove destination file " << fileName << ", aborting this file" << std::endl;
return -1;
}
}
}
else
{
string fileName = *i;
if( fileName.rfind( '/' ) != fileName.npos )
fileName = fileName.substr( fileName.rfind( '/' ) + 1 );
string destFile = localdir + '/' + fileName;
if( LocalFS::stat( destFile, s ) >= 0 )
{
cout << "Destination directory already contains file '" << fileName << "', resuming partial download." << endl;
log().debug << "Destination file already exists with size " << s.m_llSize << ", resuming partial download" << std::endl;
}
}
if (download(i->c_str(), localdir.c_str(), client, encryption) < 0)
{
// calculate total available disk size
int64_t availdisk = 0;
LocalFS::get_dir_space(localdir, availdisk);
if (availdisk <= 0)
{
// if no disk space svailable, no need to try any more
cerr << "insufficient local disk space. quit.\n";
log().error << "Insufficient disk space" << std::endl;
Utility::logout(client);
}
return -1;
}
}
}
Utility::logout(client);
log().debug << "Download completed successfully" << std::endl;
return 0;
}
int main(int argc, char** argv)
{
if (argc != 2)
{
cout << "USAGE: rm <dir>\n";
return -1;
}
Sector client;
Session s;
s.loadInfo("../conf/client.conf");
if (client.init(s.m_ClientConf.m_strMasterIP, s.m_ClientConf.m_iMasterPort) < 0)
return -1;
if (client.login(s.m_ClientConf.m_strUserName, s.m_ClientConf.m_strPassword, s.m_ClientConf.m_strCertificate.c_str()) < 0)
return -1;
string path = argv[1];
bool wc = WildCard::isWildCard(path);
if (!wc)
{
int r = client.remove(path);
if (r == SectorError::E_NOEMPTY)
{
if (isRecursive(path))
client.rmr(path);
}
else if (r < 0)
cout << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl;
}
else
{
string orig = path;
size_t p = path.rfind('/');
if (p == string::npos)
path = "/";
else
{
path = path.substr(0, p);
orig = orig.substr(p + 1, orig.length() - p);
}
vector<SNode> filelist;
int r = client.list(path, filelist);
if (r < 0)
cout << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl;
bool recursive = false;
vector<string> filtered;
for (vector<SNode>::iterator i = filelist.begin(); i != filelist.end(); ++ i)
{
if (WildCard::match(orig, i->m_strName))
{
if (recursive)
client.rmr(path + "/" + i->m_strName);
else
{
r = client.remove(path + "/" + i->m_strName);
if (r == SectorError::E_NOEMPTY)
{
recursive = isRecursive(path + "/" + i->m_strName);
if (recursive)
client.rmr(path + "/" + i->m_strName);
}
else if (r < 0)
cout << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl;
}
}
}
}
client.logout();
client.close();
return 1;
}
