/*
* Copy a file from the local filesystem to Sector.
*
* src is the path to the local file.
* dest is the path to the new Sector file.
*
* Returns: on success a non-negative value. A negative value on failure.
*/
static PyObject* upload( PyObject* self, PyObject* args )
{
SectorFile f;
const char* src;
const char* dest;
if( !PyArg_ParseTuple( args, "ss", &src, &dest ) ) {
return( NULL );
}
int status = f.open( dest, SF_MODE::WRITE );
if( status < 0 ) {
PyErr_SetString( PyExc_IOError, "failed to open destination, status=" +
status );
return( NULL );
}
status = f.upload( src, false );
if( status < 0 ) {
PyErr_SetString( PyExc_IOError, "upload failed, status=" + status );
}
f.close();
return( Py_BuildValue( "i", status ) );
}
/*
* Copy a file from Sector to the local filesystem.
*
* src is path to Sector file.
* dest is path to new local file.
*
* Returns: on success, a non-negative value. On failure, a negative value.
*/
static PyObject* download( PyObject* self, PyObject* args )
{
int status = 0;
SectorFile f;
const char* src;
const char* dest;
if( !PyArg_ParseTuple( args, "ss", &src, &dest ) ) {
return( NULL );
}
status = f.open( src, SF_MODE::READ );
if( status < 0 ) {
PyErr_SetString( PyExc_IOError, "failed to open source, status=" +
status );
return( NULL );
}
status = f.download( dest );
if( status < 0 ) {
PyErr_SetString( PyExc_IOError, "download failed, status=" + status );
}
f.close();
return( Py_BuildValue( "i", status ) );
}
/*
* Read data from a Sector file. This function can be called repeatedly
* to read through a file in chunks.
*
* filehandle is the value returned by open().
* len is the number of bytes to read from the file.
*
* Returns: on success the data read. None on EOF. On failure NULL.
*/
static PyObject* read( PyObject* self, PyObject* args )
{
SectorFile* f;
long filehandle;
long len;
if( !PyArg_ParseTuple( args, "ll", &filehandle, &len ) ) {
return( NULL );
}
f = (SectorFile*)filehandle;
char buf[len];
int status = f->read( buf, len );
if( status == 0 ) {
return( Py_None );
}
if( status < 0 ) {
PyErr_SetString( PyExc_IOError, "read failed, status=" + status );
return( NULL );
}
buf[status] = '\0';
return( Py_BuildValue( "s", buf ) );
}
int job_service::upload ( std::string const &dir, std::vector<std::string> &files )
{
int rc = saga_sectorsphere::error::SAGA_SECSP_E_SUCCESS ;
SNode attr ;
rc = Sector::stat( dir, attr ) ;
if( rc < saga_sectorsphere::error::SAGA_SECSP_E_SUCCESS || !attr.m_bIsDir ) {
if( rc = Sector::mkdir( dir ) < saga_sectorsphere::error::SAGA_SECSP_E_SUCCESS ) {
return rc ;
}
}
std::string to_create ;
std::string to_upload ;
to_create = dir ;
size_t pos = dir.rfind("/") ;
if( pos == std::string::npos ){
to_create = dir ;
}
else if( pos == to_create.size()) {
size_t p = to_create.find_last_of( "/", to_create.size() - 1 ) ;
if ( p != std::string::npos ) {
to_create = dir.substr( p, dir.size() - 1 ) ;
}
}
else{
to_create = dir.substr( pos + 1, std::string::npos ) ;
}
to_upload = dir ;
for( std::vector<std::string>::iterator it = files.begin() ; it !=files.end() ; ++it ){
SectorFile f ;
std::string cr = to_create ;
cr += "/" ;
cr += *it ;
std::string up = to_upload ;
up += "/" ;
up += *it ;
rc = f.open( cr, SF_MODE::WRITE ) ;
if( rc < saga_sectorsphere::error::SAGA_SECSP_E_SUCCESS ) {
return rc ;
}
if( rc = f.upload( up.c_str() ) < saga_sectorsphere::error::SAGA_SECSP_E_SUCCESS ){
return rc ;
}
input_fileset.push_back( cr ) ;
f.close() ;
}
return rc ;
}
int upload(const char* file, const char* dst, Sector& client)
{
CTimer timer;
uint64_t t1 = timer.getTime(); // returns time in microseconds (usecs)
struct stat s;
stat(file, &s);
cout << "uploading " << file << " of " << s.st_size << " bytes" << endl;
SectorFile* f = client.createSectorFile();
if (f->open(dst, SF_MODE::WRITE) < 0)
{
cout << "ERROR: unable to connect to server or file already exists." << endl;
return -1;
}
bool finish = true;
if (f->upload(file) < 0LL)
finish = false;
f->close();
client.releaseSectorFile(f);
if (finish)
{
float throughput = s.st_size * 8.0f / 1000000.0f / ((timer.getTime() - t1) / 1000000.0f);
cout << "Uploading accomplished! " << "AVG speed " << throughput << " Mb/s." << endl << endl ;
}
else
cout << "Uploading failed! Please retry. " << endl << endl;
return 1;
}
/*
* Close a file in Sector.
*
* filehandle is the value returned by open().
*
* Returns: on success a non-negative value. A negative value on failure.
*/
static PyObject* close( PyObject* self, PyObject* args )
{
SectorFile *f;
long filehandle;
if( !PyArg_ParseTuple( args, "l", &filehandle ) ) {
return( NULL );
}
f = (SectorFile*)filehandle;
int status = f->close();
return( Py_BuildValue( "i", status ) );
}
/*
* Open a file in Sector. Returns a long representing a pointer to
* the Sector filehandle. This pointer needs to be passed to subsequent
* file operations.
*
* path is the full path of the file to be opened.
* mode is READ ONLY (1), WRITE ONLY (2), or READ/WRITE (3).
*
* Returns: a long representing a pointer to the Sector filehandle.
*/
static PyObject* open( PyObject* self, PyObject* args )
{
const char* path;
int16_t mode;
SectorFile* f = new SectorFile();
if( !PyArg_ParseTuple( args, "si", &path, &mode ) ) {
return( NULL );
}
int status = f->open( path, mode );
if( status < 0 ) {
PyErr_SetString( PyExc_IOError, "open failed, status=" + status );
return( NULL );
}
return( Py_BuildValue( "l", f ) );
}
/*
* Write data to a Sector file.
*/
static PyObject* write( PyObject* self, PyObject* args )
{
SectorFile* f;
long filehandle;
const char* data;
long len;
if( !PyArg_ParseTuple( args, "lsl", &filehandle, &data, &len ) ) {
return( NULL );
}
f = (SectorFile*)filehandle;
int status = f->write( data, len );
if( status < 0 ) {
PyErr_SetString( PyExc_IOError, "write failed, status=" + status );
return( NULL );
}
return( Py_BuildValue( "i", status ) );
}
int upload(const char* file, const char* dst, Sector& client)
{
timeval t1, t2;
gettimeofday(&t1, 0);
struct stat64 s;
stat64(file, &s);
cout << "uploading " << file << " of " << s.st_size << " bytes" << endl;
SectorFile* f = client.createSectorFile();
if (f->open(dst, SF_MODE::WRITE) < 0)
{
cout << "ERROR: unable to connect to server or file already exists." << endl;
return -1;
}
bool finish = true;
if (f->upload(file) < 0LL)
finish = false;
f->close();
client.releaseSectorFile(f);
if (finish)
{
gettimeofday(&t2, 0);
float throughput = s.st_size * 8.0 / 1000000.0 / ((t2.tv_sec - t1.tv_sec) + (t2.tv_usec - t1.tv_usec) / 1000000.0);
cout << "Uploading accomplished! " << "AVG speed " << throughput << " Mb/s." << endl << endl ;
}
else
cout << "Uploading failed! Please retry. " << endl << endl;
return 1;
}
int upload(const char* file, const char* dst, Sector& client, const int rep_num, const string& ip, const string& cid, const bool secure, const bool smart)
{
//check if file already exists
SNode s;
if (LocalFS::stat(file, s) < 0)
{
cout << "cannot locate source file " << file << endl;
return -1;
}
SNode attr;
if (client.stat(dst, attr) >= 0)
{
if (attr.m_llSize == s.m_llSize && smart)
{
cout << "destination file " << dst << " exists on Sector FS. skip.\n";
return 0;
}
}
cout << "uploading " << file << " of " << s.m_llSize << " bytes" << endl;
timeval t1, t2;
gettimeofday(&t1, 0);
SectorFile* f = client.createSectorFile();
SF_OPT option;
option.m_llReservedSize = s.m_llSize;
if (option.m_llReservedSize <= 0)
option.m_llReservedSize = 1;
option.m_iReplicaNum = rep_num;
option.m_strHintIP = ip;
option.m_strCluster = cid;
int mode = SF_MODE::WRITE | SF_MODE::TRUNC;
if (secure)
mode |= SF_MODE::SECURE;
int r = f->open(dst, mode, &option);
if (r < 0)
{
cerr << "unable to open file " << dst << endl;
Utility::print_error(r);
return -1;
}
int64_t result = f->upload(file);
f->close();
client.releaseSectorFile(f);
if (result >= 0)
{
gettimeofday(&t2, 0);
float throughput = s.m_llSize * 8.0 / 1000000.0 / ((t2.tv_sec - t1.tv_sec) + (t2.tv_usec - t1.tv_usec) / 1000000.0);
cout << "Uploading accomplished! " << "AVG speed " << throughput << " Mb/s." << endl << endl ;
}
else
{
cout << "Uploading failed! Please retry. " << endl << endl;
Utility::print_error(result);
return result;
}
return 0;
}
int download(const char* file, const char* dest, Sector& client, bool encryption)
{
#ifndef WIN32
timeval t1, t2;
#else
DWORD t1, t2;
#endif
#ifndef WIN32
gettimeofday(&t1, 0);
#else
t1 = GetTickCount();
#endif
SNode attr;
int rc;
if ((rc = client.stat(file, attr)) < 0)
{
cerr << "ERROR: cannot locate file " << file << endl;
log().error << "stat of source file " << file << " failed with rc = " << rc << std::endl;
return -1;
}
if (attr.m_bIsDir)
{
rc = ::mkdir((string(dest) + "/" + file).c_str(), S_IRWXU);
if( rc < 0 && errno == EEXIST )
{
log().debug << "directory " << dest << '/' << file << " already exists" << std::endl;
return 1;
}
else if( rc < 0 )
{
int errno_save = errno;
log().error << "Failed to create directory " << dest << '/' << file << ", errno = " << errno_save << std::endl;
return -1;
}
else
return 1;
}
const long long int size = attr.m_llSize;
cout << "downloading " << file << " of " << size << " bytes" << endl;
log().debug << "Downloading " << file << " of " << size << " bytes" << std::endl;
SectorFile* f = client.createSectorFile();
int mode = SF_MODE::READ;
if (encryption)
mode |= SF_MODE::SECURE;
if ((rc = f->open(file, mode)) < 0)
{
cerr << "unable to locate file " << file << endl;
log().error << "Failed to open sector file " << file << " with err = " << rc << std::endl;
return -1;
}
int sn = strlen(file) - 1;
for (; sn >= 0; sn --)
{
if (file[sn] == '/')
break;
}
string localpath;
if (dest[strlen(dest) - 1] != '/')
localpath = string(dest) + string("/") + string(file + sn + 1);
else
localpath = string(dest) + string(file + sn + 1);
log().debug << "Downloading " << file << " to " << localpath << std::endl;
int64_t result = f->download(localpath.c_str(), true);
f->close();
client.releaseSectorFile(f);
if (result >= 0)
{
float throughput = 0.0;
#ifndef WIN32
gettimeofday(&t2, 0);
float span = (t2.tv_sec - t1.tv_sec) + (t2.tv_usec - t1.tv_usec) / 1000000.0;
#else
float span = (GetTickCount() - t1) / 1000.0;
#endif
if (span > 0.0)
throughput = result * 8.0 / 1000000.0 / span;
cout << "Downloading accomplished! " << "AVG speed " << throughput << " Mb/s." << endl << endl ;
log().debug << "Download of file successful!" << std::endl;
return 0;
}
cerr << "error happened during downloading " << file << endl;
log().debug << "Failed to download file, res = " << result << std::endl;
Utility::print_error(result);
return -1;
}
int main(int argc, char** argv)
{
CmdLineParser clp;
if ((clp.parse(argc, argv) < 0) || (clp.m_mDFlags.size() != 1))
{
cerr << "usage #1: <your_application> | sector_pipe -d dst_file" << endl;
cerr << "usage #2: sector_pipe -s src_file | <your_application>" << endl;
return 0;
}
string option = clp.m_mDFlags.begin()->first;
Sector client;
if (Utility::login(client) < 0)
return -1;
timeval t1, t2;
gettimeofday(&t1, 0);
int64_t total_size = 0;
SectorFile* f = client.createSectorFile();
if (option == "d")
{
if (f->open(argv[2], SF_MODE::WRITE | SF_MODE::APPEND) < 0)
{
cerr << "ERROR: unable to open destination file." << endl;
return -1;
}
int size = 1000000;
char* buf = new char[size];
int read_size = size;
// read data from sdtin and write to Sector
while(true)
{
read_size = read(0, buf, size);
if (read_size <= 0)
break;
f->write(buf, read_size);
total_size += read_size;
}
delete [] buf;
}
else if (option == "s")
{
if (f->open(argv[2], SF_MODE::READ) < 0)
{
cerr << "ERROR: unable to open source file." << endl;
return -1;
}
int size = 1000000;
char* buf = new char[size];
int read_size = size;
// read data from Sector and write to stdout
while(!f->eof())
{
read_size = f->read(buf, size);
if (read_size <= 0)
break;
total_size += read_size;
write(1, buf, read_size);
}
delete [] buf;
}
f->close();
client.releaseSectorFile(f);
gettimeofday(&t2, 0);
double throughput = total_size * 8.0 / 1000000.0 / ((t2.tv_sec - t1.tv_sec) + (t2.tv_usec - t1.tv_usec) / 1000000.0);
cerr << "Pipeline accomplished! " << "AVG speed " << throughput << " Mb/s." << endl << endl ;
Utility::logout(client);
return 0;
}
int upload(const char* file, const char* dst, Sector& client)
{
//check if file already exists
struct stat st;
if (stat(file, &st) < 0)
{
cout << "cannot locate source file " << file << endl;
return -1;
}
SNode attr;
if (client.stat(dst, attr) >= 0)
{
if (attr.m_llSize == st.st_size)
{
cout << "destination file " << dst << " exists on Sector FS. skip.\n";
return 0;
}
}
CTimer timer;
uint64_t t1 = timer.getTime(); // returns time in microseconds (usecs)
struct stat s;
stat(file, &s);
cout << "uploading " << file << " of " << s.st_size << " bytes" << endl;
SectorFile* f = client.createSectorFile();
int64_t reserve = s.st_size;
if (reserve <= 0)
reserve = 1;
int r = f->open(dst, SF_MODE::WRITE, "", reserve);
if (r < 0)
{
cerr << "unable to open file " << dst << endl;
Utility::print_error(r);
return -1;
}
int64_t result = f->upload(file);
f->close();
client.releaseSectorFile(f);
if (result >= 0)
{
float throughput = s.st_size * 8.0f / 1000000.0f / ((timer.getTime() - t1) / 1000000.0f);
cout << "Uploading accomplished! " << "AVG speed " << throughput << " Mb/s." << endl << endl ;
}
else
{
cout << "Uploading failed! Please retry. " << endl << endl;
Utility::print_error(result);
return -1;
}
return 0;
}
