/** For SwarmPlayer 3000's HTTP failover. We should exit if swift isn't
* delivering such that the extension can start talking HTTP to the backup.
*/
bool HTTPIsSending()
{
if (http_gw_reqs_open > 0)
{
FileTransfer *ft = FileTransfer::file(http_requests[http_gw_reqs_open-1].transfer);
if (ft != NULL) {
fprintf(stderr,"httpgw: upload %lf\n",ft->GetCurrentSpeed(DDIR_UPLOAD)/1024.0);
fprintf(stderr,"httpgw: dwload %lf\n",ft->GetCurrentSpeed(DDIR_DOWNLOAD)/1024.0);
dprintf("httpgw: upload %lf\n",ft->GetCurrentSpeed(DDIR_UPLOAD)/1024.0);
dprintf("httpgw: dwload %lf\n",ft->GetCurrentSpeed(DDIR_DOWNLOAD)/1024.0);
}
}
return true;
// TODO: reactivate when used in SwiftTransport / SwarmPlayer 3000.
if (test_time == 0)
{
test_time = NOW;
return true;
}
if (NOW > test_time+5*1000*1000)
{
fprintf(stderr,"http alive: httpc count is %d\n", http_gw_reqs_open );
if (http_gw_reqs_open == 0 && !sawhttpconn)
{
fprintf(stderr,"http alive: no HTTP activity ever, quiting\n");
return false;
}
else
sawhttpconn = true;
for (int httpc=0; httpc<http_gw_reqs_open; httpc++)
{
/*
if (http_requests[httpc].offset >= 100000)
{
fprintf(stderr,"http alive: 100K sent, quit\n");
return false;
}
else
{
fprintf(stderr,"http alive: sent %lli\n", http_requests[httpc].offset );
return true;
}
*/
// If
// a. don't know anything about content (i.e., size still 0) or
// b. not sending to HTTP client and not at end, and
// not downloading from P2P and not at end
// then stop.
if ( swift::Size(http_requests[httpc].transfer) == 0 || \
(http_requests[httpc].offset == lastoffset &&
http_requests[httpc].offset != swift::Size(http_requests[httpc].transfer) && \
swift::Complete(http_requests[httpc].transfer) == lastcomplete && \
swift::Complete(http_requests[httpc].transfer) != swift::Size(http_requests[httpc].transfer)))
{
fprintf(stderr,"http alive: no progress, quiting\n");
//getchar();
return false;
}
/*
if (http_requests[httpc].offset == swift::Size(http_requests[httpc].transfer))
{
// TODO: seed for a while.
fprintf(stderr,"http alive: data delivered to client, quiting\n");
return false;
}
*/
lastoffset = http_requests[httpc].offset;
lastcomplete = swift::Complete(http_requests[httpc].transfer);
}
test_time = NOW;
return true;
}
else
return true;
}
void ZeroState::LibeventCleanCallback(int fd, short event, void *arg)
{
//fprintf(stderr,"zero clean: enter\n");
// Arno, 2012-02-24: Why-oh-why, update NOW
Channel::Time();
ZeroState *zs = (ZeroState *)arg;
if (zs == NULL)
return;
// See which zero state FileTransfers have no clients
std::set<FileTransfer *> delset;
for(int i=0; i<FileTransfer::files.size(); i++)
{
FileTransfer *ft = FileTransfer::files[i];
if (ft == NULL)
continue;
if (!ft->IsZeroState())
continue;
// Arno, 2012-07-20: Some weirdness on Win7 when we use GetChannels()
// all the time. Map/set iterators incompatible?!
channels_t channels = ft->GetChannels();
if (channels.size() == 0)
{
// Ain't go no clients, cleanup transfer.
delset.insert(ft);
}
else if (zs->connect_timeout_ != TINT_NEVER)
{
// Garbage collect really slow connections, essential on Mac.
dprintf("%s zero clean %s has %d peers\n",tintstr(),ft->root_hash().hex().c_str(), ft->GetChannels().size() );
channels_t::iterator iter2;
for (iter2=channels.begin(); iter2!=channels.end(); iter2++) {
Channel *c = *iter2;
if (c != NULL)
{
//fprintf(stderr,"%s F%u zero clean %s opentime %lld connect %lld\n",tintstr(),ft->fd(), c->peer().str(), (NOW-c->GetOpenTime()), zs->connect_timeout_ );
// Garbage collect channels when open for long and slow upload
if ((NOW-c->GetOpenTime()) > zs->connect_timeout_)
{
//fprintf(stderr,"%s F%u zero clean %s opentime %lld ulspeed %lf\n",tintstr(),ft->fd(), c->peer().str(), (NOW-c->GetOpenTime())/TINT_SEC, ft->GetCurrentSpeed(DDIR_UPLOAD) );
fprintf(stderr,"%s F%u zero clean %s close slow channel\n",tintstr(),ft->fd(), c->peer().str() );
c->Close();
delete c;
}
}
}
if (ft->GetChannels().size() == 0)
{
// Ain't go no clients left, cleanup transfer.
delset.insert(ft);
}
}
}
// Delete 0-state FileTransfers sans peers
std::set<FileTransfer *>::iterator iter;
for (iter=delset.begin(); iter!=delset.end(); iter++)
{
FileTransfer *ft = *iter;
dprintf("%s F%u zero clean close\n",tintstr(),ft->fd() );
//fprintf(stderr,"%s F%u zero clean close\n",tintstr(),ft->fd() );
swift::Close(ft->fd());
}
// Reschedule cleanup
evtimer_add(&(zs->evclean_),tint2tv(CLEANUP_INTERVAL*TINT_SEC));
}
void HttpGwMayWriteCallback (int transfer) {
// Write some data to client
http_gw_t* req = HttpGwFindRequestByTransfer(transfer);
if (req == NULL) {
print_error("httpgw: MayWrite: can't find req for transfer");
return;
}
uint64_t complete = swift::SeqComplete(req->transfer);
dprintf("%s @%i http write complete %lli offset %lli\n",tintstr(),req->id, complete, req->offset);
//fprintf(stderr,"offset %lli seqcomp %lli comp %lli\n",req->offset, complete, swift::Complete(req->transfer) );
struct evhttp_connection *evconn = evhttp_request_get_connection(req->sinkevreq);
struct bufferevent* buffy = evhttp_connection_get_bufferevent(evconn);
struct evbuffer *outbuf = bufferevent_get_output(buffy);
//fprintf(stderr,"httpgw: MayWrite avail %i bufev outbuf %i\n",complete-req->offset, evbuffer_get_length(outbuf) );
dprintf("httpgw: MayWrite avail %i bufev outbuf %i\n",complete-req->offset, evbuffer_get_length(outbuf) );
if (complete > req->offset && evbuffer_get_length(outbuf) < HTTPGW_MAX_BUFFER_BYTES)
{
// Received more than I pushed to player, send data
char buf[HTTPGW_MAX_WRITE_BYTES];
// Arno, 2010-08-16, TODO
#ifdef WIN32
uint64_t tosend = min(HTTPGW_MAX_WRITE_BYTES,complete-req->offset);
#else
uint64_t tosend = std::min((uint64_t)HTTPGW_MAX_WRITE_BYTES,complete-req->offset);
#endif
size_t rd = pread(req->transfer,buf,tosend,req->offset); // hope it is cached
if (rd<0) {
print_error("httpgw: MayWrite: error pread");
HttpGwCloseConnection(req);
return;
}
// Construct evbuffer and send incrementally
struct evbuffer *evb = evbuffer_new();
int ret = evbuffer_add(evb,buf,rd);
if (ret < 0) {
print_error("httpgw: MayWrite: error evbuffer_add");
evbuffer_free(evb);
HttpGwCloseConnection(req);
return;
}
if (req->offset == 0) {
// Not just for chunked encoding, see libevent2's http.c
evhttp_send_reply_start(req->sinkevreq, 200, "OK");
}
evhttp_send_reply_chunk(req->sinkevreq, evb);
evbuffer_free(evb);
int wn = rd;
dprintf("%s @%i http sent data %ib\n",tintstr(),req->id,(int)wn);
req->offset += wn;
req->tosend -= wn;
// PPPLUG
FileTransfer *ft = FileTransfer::file(transfer);
if (ft == NULL)
return;
ft->picker().updatePlaybackPos( wn/ft->file().chunk_size() );
}
// Arno, 2010-11-30: tosend is set to fuzzy len, so need extra/other test.
if (req->tosend==0 || req->offset == swift::Size(req->transfer)) {
// done; wait for new HTTP request
dprintf("%s @%i done\n",tintstr(),req->id);
//fprintf(stderr,"httpgw: MayWrite: done, wait for buffer empty before send_end_reply\n" );
if (evbuffer_get_length(outbuf) == 0) {
//fprintf(stderr,"httpgw: MayWrite: done, buffer empty, end req\n" );
HttpGwCloseConnection(req);
}
}
else {
// wait for data
dprintf("%s @%i waiting for data\n",tintstr(),req->id);
}
}
void HttpGwNewRequestCallback (struct evhttp_request *evreq, void *arg) {
dprintf("%s @%i http new request\n",tintstr(),http_gw_reqs_count+1);
if (evhttp_request_get_command(evreq) != EVHTTP_REQ_GET) {
return;
}
sawhttpconn = true;
// Parse URI
const char *uri = evhttp_request_get_uri(evreq);
//struct evkeyvalq *headers = evhttp_request_get_input_headers(evreq);
//const char *contentrangestr =evhttp_find_header(headers,"Content-Range");
char *tokenuri = (char *)malloc(strlen(uri)+1);
strcpy(tokenuri,uri);
char * hashch=strtok(tokenuri,"/"), hash[41];
while (hashch && (1!=sscanf(hashch,"%40[0123456789abcdefABCDEF]",hash) || strlen(hash)!=40))
hashch = strtok(NULL,"/");
free(tokenuri);
if (strlen(hash)!=40) {
evhttp_send_error(evreq,400,"Path must be root hash in hex, 40 bytes.");
return;
}
char *xcontentdur = NULL;
if (strlen(uri) > 42) {
xcontentdur = (char *)malloc(strlen(uri)-42+1);
strcpy(xcontentdur,&uri[42]);
}
else
xcontentdur = (char *)"0";
dprintf("%s @%i demands %s %s\n",tintstr(),http_gw_reqs_open+1,hash,xcontentdur);
std::string filenamestr = "";
if (httpgw_storage_dir == NULL) {
filenamestr = hash;
}
else {
filenamestr = httpgw_storage_dir;
filenamestr += hash;
}
dprintf("%s @%i http destination %s\n",tintstr(),http_gw_reqs_open+1, filenamestr.c_str() );
// initiate transmission
Sha1Hash root_hash = Sha1Hash(true,hash);
int transfer = swift::Find(root_hash);
if (transfer==-1) {
transfer = swift::Open(filenamestr.c_str(),root_hash,Address(),false,httpgw_chunk_size); // ARNOTODO: allow for chunk size to be set via URL?
dprintf("%s @%i trying to HTTP GET swarm %s that has not been STARTed speeds %lf/%lf\n",tintstr(),http_gw_reqs_open+1,hash, httpgw_maxspeed[DDIR_DOWNLOAD], httpgw_maxspeed[DDIR_UPLOAD]);
// Arno, 2011-12-20: Only on new transfers, otherwise assume that CMD GW
// controls speed
FileTransfer *ft = FileTransfer::file(transfer);
ft->SetMaxSpeed(DDIR_DOWNLOAD,httpgw_maxspeed[DDIR_DOWNLOAD]);
ft->SetMaxSpeed(DDIR_UPLOAD,httpgw_maxspeed[DDIR_UPLOAD]);
}
// Record request
http_gw_t* req = http_requests + http_gw_reqs_open++;
req->id = ++http_gw_reqs_count;
req->sinkevreq = evreq;
req->xcontentdur = xcontentdur;
req->offset = 0;
req->tosend = 0;
req->transfer = transfer;
req->lastcpoffset = 0;
req->sinkevwrite = NULL;
req->closing = false;
fprintf(stderr,"httpgw: Opened %s\n",hash);
// We need delayed replying, so take ownership.
// See http://code.google.com/p/libevent-longpolling/source/browse/trunk/main.c
// Careful: libevent docs are broken. It doesn't say that evhttp_send_reply_send
// actually calls evhttp_request_free, i.e. releases ownership for you.
//
evhttp_request_own(evreq);
// Register callback for connection close
struct evhttp_connection *evconn = evhttp_request_get_connection(req->sinkevreq);
evhttp_connection_set_closecb(evconn,HttpGwLibeventCloseCallback,req->sinkevreq);
if (swift::Size(transfer)) {
HttpGwFirstProgressCallback(transfer,bin_t(0,0));
} else {
swift::AddProgressCallback(transfer,&HttpGwFirstProgressCallback,HTTPGW_FIRST_PROGRESS_BYTE_INTERVAL_AS_LAYER);
}
}
// upload the files associated with the jobads to the sandbox at td_sinful
// with the supplied capability.
// The work_ad should contain:
// ATTR_TREQ_CAPABILITY
// ATTR_TREQ_FTP
// ATTR_TREQ_JOBID_ALLOW_LIST
bool
DCTransferD::upload_job_files(int JobAdsArrayLen, ClassAd* JobAdsArray[],
ClassAd *work_ad, CondorError * errstack)
{
ReliSock *rsock = NULL;
int timeout = 60 * 60 * 8; // transfers take a long time...
int i;
ClassAd reqad, respad;
std::string cap;
int ftp;
int invalid;
int protocol;
std::string reason;
//////////////////////////////////////////////////////////////////////////
// Connect to the transferd and authenticate
//////////////////////////////////////////////////////////////////////////
// This call with automatically connect to _addr, which was set in the
// constructor of this object to be the transferd in question.
rsock = (ReliSock*)startCommand(TRANSFERD_WRITE_FILES, Stream::reli_sock,
timeout, errstack);
if( ! rsock ) {
dprintf( D_ALWAYS, "DCTransferD::upload_job_files: "
"Failed to send command (TRANSFERD_WRITE_FILES) "
"to the schedd\n" );
errstack->push("DC_TRANSFERD", 1,
"Failed to start a TRANSFERD_WRITE_FILES command.");
return false;
}
// First, if we're not already authenticated, force that now.
if (!forceAuthentication( rsock, errstack )) {
dprintf( D_ALWAYS, "DCTransferD::upload_job_files() authentication "
"failure: %s\n", errstack->getFullText().c_str() );
errstack->push("DC_TRANSFERD", 1,
"Failed to authenticate properly.");
return false;
}
rsock->encode();
//////////////////////////////////////////////////////////////////////////
// Query the transferd about the capability/protocol and see if I can
// upload my files. It will respond with a classad saying good or bad.
//////////////////////////////////////////////////////////////////////////
work_ad->LookupString(ATTR_TREQ_CAPABILITY, cap);
work_ad->LookupInteger(ATTR_TREQ_FTP, ftp);
reqad.Assign(ATTR_TREQ_CAPABILITY, cap);
reqad.Assign(ATTR_TREQ_FTP, ftp);
// This request ad to the transferd should contain:
// ATTR_TREQ_CAPABILITY
// ATTR_TREQ_FTP
putClassAd(rsock, reqad);
rsock->end_of_message();
rsock->decode();
// This response ad to the transferd should contain:
// ATTR_TREQ_INVALID_REQUEST (set to true)
// ATTR_TREQ_INVALID_REASON
//
// OR
//
// ATTR_TREQ_INVALID_REQUEST (set to false)
getClassAd(rsock, respad);
rsock->end_of_message();
respad.LookupInteger(ATTR_TREQ_INVALID_REQUEST, invalid);
if (invalid == TRUE) {
// The transferd rejected my attempt to upload the fileset
delete rsock;
respad.LookupString(ATTR_TREQ_INVALID_REASON, reason);
errstack->push("DC_TRANSFERD", 1, reason.c_str());
return false;
}
//////////////////////////////////////////////////////////////////////////
// Sort the job ads array into numerically sorted order. The transferd
// must do the same.
//////////////////////////////////////////////////////////////////////////
// TODO
//////////////////////////////////////////////////////////////////////////
// Based upon the protocol I've chosen, use that method to upload the
// files. When using the FileTrans protocol, a child process on the
// transferd side will be inheriting a socket and accepting the
// FileTransfer object's protocol.
//////////////////////////////////////////////////////////////////////////
// XXX Fix to only send jobads for allowed jobs
dprintf(D_ALWAYS, "Sending fileset");
work_ad->LookupInteger(ATTR_TREQ_FTP, protocol);
switch(protocol) {
case FTP_CFTP:
// upload the files using the FileTransfer Object
for (i=0; i<JobAdsArrayLen; i++) {
FileTransfer ftrans;
if ( !ftrans.SimpleInit(JobAdsArray[i], false, false, rsock) )
{
delete rsock;
errstack->push("DC_TRANSFERD", 1,
"Failed to initate uploading of files.");
return false;
}
ftrans.setPeerVersion( version() );
if ( !ftrans.UploadFiles(true,false) ) {
delete rsock;
errstack->push("DC_TRANSFERD", 1,
"Failed to upload files.");
return false;
}
dprintf(D_ALWAYS | D_NOHEADER, ".");
}
rsock->end_of_message();
dprintf(D_ALWAYS | D_NOHEADER, "\n");
break;
default:
// Bail due to user error. This client doesn't support the uknown
// protocol.
delete rsock;
errstack->push("DC_TRANSFERD", 1,
"Unknown file transfer protocol selected.");
return false;
break;
}
//////////////////////////////////////////////////////////////////////////
// Get the response from the transferd once it sees a completed
// movement of files from the child process.
//////////////////////////////////////////////////////////////////////////
rsock->decode();
getClassAd(rsock, respad);
rsock->end_of_message();
// close up shop
delete rsock;
respad.LookupInteger(ATTR_TREQ_INVALID_REQUEST, invalid);
if ( invalid == TRUE ) {
respad.LookupString(ATTR_TREQ_INVALID_REASON, reason);
errstack->push("DC_TRANSFERD", 1, reason.c_str());
return false;
}
return true;
}
// download the files associated with the jobads to the sandbox at td_sinful
// with the supplied capability.
// The work_ad should contain:
// ATTR_TREQ_CAPABILITY
// ATTR_TREQ_FTP
// ATTR_TREQ_JOBID_ALLOW_LIST
bool
DCTransferD::download_job_files(ClassAd *work_ad, CondorError * errstack)
{
ReliSock *rsock = NULL;
int timeout = 60 * 60 * 8; // transfers take a long time...
int i;
ClassAd reqad, respad;
std::string cap;
int ftp;
int invalid;
int protocol;
std::string reason;
int num_transfers;
ClassAd jad;
const char *lhstr = NULL;
ExprTree *tree = NULL;
//////////////////////////////////////////////////////////////////////////
// Connect to the transferd and authenticate
//////////////////////////////////////////////////////////////////////////
// This call with automatically connect to _addr, which was set in the
// constructor of this object to be the transferd in question.
rsock = (ReliSock*)startCommand(TRANSFERD_READ_FILES, Stream::reli_sock,
timeout, errstack);
if( ! rsock ) {
dprintf( D_ALWAYS, "DCTransferD::download_job_files: "
"Failed to send command (TRANSFERD_READ_FILES) "
"to the schedd\n" );
errstack->push("DC_TRANSFERD", 1,
"Failed to start a TRANSFERD_READ_FILES command.");
return false;
}
// First, if we're not already authenticated, force that now.
if (!forceAuthentication( rsock, errstack )) {
dprintf( D_ALWAYS, "DCTransferD::download_job_files() authentication "
"failure: %s\n", errstack->getFullText().c_str() );
errstack->push("DC_TRANSFERD", 1,
"Failed to authenticate properly.");
return false;
}
rsock->encode();
//////////////////////////////////////////////////////////////////////////
// Query the transferd about the capability/protocol and see if I can
// download my files. It will respond with a classad saying good or bad.
//////////////////////////////////////////////////////////////////////////
work_ad->LookupString(ATTR_TREQ_CAPABILITY, cap);
work_ad->LookupInteger(ATTR_TREQ_FTP, ftp);
reqad.Assign(ATTR_TREQ_CAPABILITY, cap);
reqad.Assign(ATTR_TREQ_FTP, ftp);
// This request ad to the transferd should contain:
// ATTR_TREQ_CAPABILITY
// ATTR_TREQ_FTP
putClassAd(rsock, reqad);
rsock->end_of_message();
rsock->decode();
// This response ad from the transferd should contain:
// ATTR_TREQ_INVALID_REQUEST (set to true)
// ATTR_TREQ_INVALID_REASON
//
// OR
//
// ATTR_TREQ_INVALID_REQUEST (set to false)
// ATTR_TREQ_NUM_TRANSFERS
//
getClassAd(rsock, respad);
rsock->end_of_message();
respad.LookupInteger(ATTR_TREQ_INVALID_REQUEST, invalid);
if (invalid == TRUE) {
// The transferd rejected my attempt to upload the fileset
delete rsock;
respad.LookupString(ATTR_TREQ_INVALID_REASON, reason);
errstack->push("DC_TRANSFERD", 1, reason.c_str());
return false;
}
respad.LookupInteger(ATTR_TREQ_NUM_TRANSFERS, num_transfers);
//////////////////////////////////////////////////////////////////////////
// Based upon the protocol I've chosen, use that method to download the
// files. When using the FileTrans protocol, a child process on the
// transferd side will be sending me individual job ads and then
// instantiating a filetransfer object for that ad.
//////////////////////////////////////////////////////////////////////////
dprintf(D_ALWAYS, "Receiving fileset");
work_ad->LookupInteger(ATTR_TREQ_FTP, protocol);
switch(protocol) {
case FTP_CFTP: // download the files using the FileTransfer Object
for (i = 0; i < num_transfers; i++) {
// Grab a job ad the server is sending us so we know what
// to receive.
getClassAd(rsock, jad);
rsock->end_of_message();
// translate the job ad by replacing the
// saved SUBMIT_ attributes so the download goes into the
// correct place.
jad.ResetExpr();
while( jad.NextExpr(lhstr, tree) ) {
if ( lhstr && strncasecmp("SUBMIT_",lhstr,7)==0 ) {
// this attr name starts with SUBMIT_
// compute new lhs (strip off the SUBMIT_)
const char *new_attr_name = strchr(lhstr,'_');
ExprTree * pTree;
ASSERT(new_attr_name);
new_attr_name++;
// insert attribute
pTree = tree->Copy();
jad.Insert(new_attr_name, pTree);
}
} // while next expr
// instantiate a filetransfer object and have it accept the
// files.
FileTransfer ftrans;
if ( !ftrans.SimpleInit(&jad, false, false, rsock) )
{
delete rsock;
errstack->push("DC_TRANSFERD", 1,
"Failed to initate uploading of files.");
return false;
}
// We want files to be copied to their final places, so apply
// any filename remaps when downloading.
if ( !ftrans.InitDownloadFilenameRemaps(&jad) ) {
return false;
}
ftrans.setPeerVersion( version() );
if ( !ftrans.DownloadFiles() ) {
delete rsock;
errstack->push("DC_TRANSFERD", 1,
"Failed to download files.");
return false;
}
dprintf(D_ALWAYS | D_NOHEADER, ".");
}
rsock->end_of_message();
dprintf(D_ALWAYS | D_NOHEADER, "\n");
break;
default:
// Bail due to user error. This client doesn't support the unknown
// protocol.
delete rsock;
errstack->push("DC_TRANSFERD", 1,
"Unknown file transfer protocol selected.");
return false;
break;
}
//////////////////////////////////////////////////////////////////////////
// Get the response from the transferd once it sees a completed
// movement of files to the child process.
//////////////////////////////////////////////////////////////////////////
rsock->decode();
getClassAd(rsock, respad);
rsock->end_of_message();
// close up shop
delete rsock;
respad.LookupInteger(ATTR_TREQ_INVALID_REQUEST, invalid);
if ( invalid == TRUE ) {
respad.LookupString(ATTR_TREQ_INVALID_REASON, reason);
errstack->push("DC_TRANSFERD", 1, reason.c_str());
return false;
}
return true;
}
void swift::ExternallyRetrieved (int transfer,bin64_t piece) {
FileTransfer* trans = FileTransfer::file(transfer);
if (!trans)
return;
trans->ack_out().set(piece); // that easy
}
int swift:: Find (Sha1Hash hash) {
FileTransfer* t = FileTransfer::Find(hash);
if (t)
return t->fd();
return -1;
}
int do_command_download_sandbox(void *arg, Stream*) {
dprintf(D_ALWAYS, "FTGAHP: download sandbox\n");
Gahp_Args args;
parse_gahp_command ((char*)arg, &args);
// first two args: result id and sandbox id:
std::string rid = args.argv[1];
std::string sid = args.argv[2];
// third arg: job ad
ClassAd ad;
classad::ClassAdParser my_parser;
if (!(my_parser.ParseClassAd(args.argv[3], ad))) {
// FAIL
write_to_pipe( ChildErrorPipe, "Failed to parse job ad" );
return 1;
}
// first, create the directory that will be IWD. returns the
// directory created.
std::string iwd;
bool success;
success = create_sandbox_dir(sid, iwd);
if (!success) {
// FAIL
write_to_pipe( ChildErrorPipe, "Failed to create sandbox" );
return 1;
}
// rewrite the IWD to the newly created sandbox dir
ad.Assign(ATTR_JOB_IWD, iwd.c_str());
char ATTR_SANDBOX_ID[] = "SandboxId";
ad.Assign(ATTR_SANDBOX_ID, sid.c_str());
// directory was created, let's set up the FileTransfer object
FileTransfer ft;
if (!ft.Init(&ad)) {
// FAIL
write_to_pipe( ChildErrorPipe, "Failed to initialize FileTransfer" );
return 1;
}
// lookup ATTR_VERSION and set it. this changes the wire
// protocol and it is important that this happens before
// calling DownloadFiles.
char* peer_version = NULL;
ad.LookupString(ATTR_VERSION, &peer_version);
ft.setPeerVersion(peer_version);
free (peer_version);
dprintf(D_ALWAYS, "BOSCO: calling Download files\n");
// the "true" param to DownloadFiles here means blocking (i.e. "in the foreground")
if (!ft.DownloadFiles(true)) {
// FAIL
write_to_pipe( ChildErrorPipe, ft.GetInfo().error_desc.Value() );
return 1;
}
// SUCCEED
return 0;
}
int do_command_upload_sandbox(void *arg, Stream*) {
dprintf(D_ALWAYS, "FTGAHP: upload sandbox\n");
Gahp_Args args;
parse_gahp_command ((char*)arg, &args);
// first two args: result id and sandbox id:
std::string rid = args.argv[1];
std::string sid = args.argv[2];
// third arg: job ad
ClassAd ad;
classad::ClassAdParser my_parser;
if (!(my_parser.ParseClassAd(args.argv[3], ad))) {
// FAIL
write_to_pipe( ChildErrorPipe, "Failed to parse job ad" );
return 1;
}
// rewrite the IWD to the actual sandbox dir
std::string iwd;
define_sandbox_path(sid, iwd);
ad.Assign(ATTR_JOB_IWD, iwd.c_str());
char ATTR_SANDBOX_ID[] = "SandboxId";
ad.Assign(ATTR_SANDBOX_ID, sid.c_str());
// directory was created, let's set up the FileTransfer object
FileTransfer ft;
if (!ft.Init(&ad)) {
// FAIL
write_to_pipe( ChildErrorPipe, "Failed to initialize FileTransfer" );
return 1;
}
// Set the Condor version of our peer, as given by the CONDOR_VERSION
// command.
// If we don't have a version, then assume it's pre-8.1.0.
// In 8.1, the file transfer protocol changed and we added
// the CONDOR_VERSION command.
if ( !peer_condor_version.empty() ) {
ft.setPeerVersion( peer_condor_version.c_str() );
} else {
CondorVersionInfo ver( 8, 0, 0 );
ft.setPeerVersion( ver );
}
if ( !sec_session_id.empty() ) {
ft.setSecuritySession( sec_session_id.c_str() );
}
dprintf(D_ALWAYS, "BOSCO: calling upload files\n");
// the "true" param to UploadFiles here means blocking (i.e. "in the foreground")
if (!ft.UploadFiles(true)) {
// FAIL
write_to_pipe( ChildErrorPipe, ft.GetInfo().error_desc.Value() );
return 1;
}
// SUCCEED
return 0;
}
void FileTransferDlgNotify::createFile(const QString &name, unsigned size, bool bCanResume)
{
m_name = name;
m_size = size;
m_name = m_name.replace('\\', '/');
FileTransfer *ft = m_dlg->m_msg->m_transfer;
int n = m_name.lastIndexOf('/');
if (n >= 0)
{
QString path;
QString p(m_name.left(n));
m_fn = m_name.right(m_name.length()-n);
m_fn = m_fn.replace(QRegExp("/"), "");
while (!p.isEmpty())
{
QString pp = getToken(p, '/');
if (!path.isEmpty())
path += '/';
if (pp == "..")
{
QString errMsg = i18n("Bad path: %1") .arg(m_name);
m_dlg->m_msg->setError(errMsg);
ft->setError();
return;
}
path += pp;
QDir dd(ft->dir() /* + '/' + path */);
QDir d(ft->dir());
if (dd.exists() || d.mkdir(path))
continue;
QString errMsg = i18n("Can't create: %1") .arg(path);
m_dlg->m_msg->setError(errMsg);
ft->setError();
return;
}
}
m_dlg->m_msg->addFile(m_name, size);
if (m_name.isEmpty() || m_name[(int)(m_name.length() - 1)] == '/')
{
ft->startReceive(0);
return;
}
QString shortName = m_name;
//m_name = ft->dir() + m_name; Quickfix, noragen
if (m_fn.isEmpty())
m_fn=m_name;
m_name = ft->dir() + m_fn;
if (ft->m_file)
delete ft->m_file;
m_dlg->process();
ft->m_file = new QFile(m_name);
if (!ft->m_file->exists())
{
if (ft->m_file->open(QIODevice::WriteOnly))
{
ft->startReceive(0);
return;
}
}
else switch (ft->overwrite())
{
case Skip:
skip();
return;
case Replace:
if (ft->m_file->open(QIODevice::WriteOnly | QIODevice::Truncate))
{
ft->startReceive(0);
return;
}
break;
case Resume:
if (ft->m_file->open(QIODevice::WriteOnly | QIODevice::Append))
{
resume();
return;
}
break;
default:
if (ft->m_file->open(QIODevice::WriteOnly | QIODevice::Append))
{
QStringList buttons;
QString forAll;
if (ft->files())
forAll = i18n("For all files");
buttons.append(i18n("&Replace"));
buttons.append(i18n("&Skip"));
if (bCanResume && (ft->m_file->size() < size))
buttons.append(i18n("Resu&me"));
m_dlg->m_ask = new BalloonMsg(NULL, quoteString(i18n("File %1 exists") .arg(shortName)),
buttons, m_dlg->lblState, NULL, false, true, 150, forAll);
QObject::connect(m_dlg->m_ask, SIGNAL(action(int, void*)), m_dlg, SLOT(action(int, void*)));
raiseWindow(m_dlg);
m_dlg->m_ask->show();
return;
}
}
bool FileServerHandler::HandleQueryFileTransfer(SocketHandler *socket,
QStringList &commands, QStringList &slist)
{
if (commands.size() != 2)
return false;
if (slist.size() < 2)
return false;
QStringList res;
int recnum = commands[1].toInt();
FileTransfer *ft;
{
QReadLocker rlock(&m_ftLock);
if (!m_ftMap.contains(recnum))
{
if (slist[1] == "DONE")
res << "ok";
else
{
LOG(VB_GENERAL, LOG_ERR,
QString("Unknown file transfer socket: %1").arg(recnum));
res << "ERROR"
<< "unknown_file_transfer_socket";
}
socket->SendStringList(res);
return true;
}
ft = m_ftMap[recnum];
ft->UpRef();
}
if (slist[1] == "IS_OPEN")
{
res << QString::number(ft->isOpen());
}
else if (slist[1] == "DONE")
{
ft->Stop();
res << "ok";
}
else if (slist[1] == "REQUEST_BLOCK")
{
if (slist.size() != 3)
{
LOG(VB_GENERAL, LOG_ERR, "Invalid QUERY_FILETRANSFER "
"REQUEST_BLOCK call");
res << "ERROR" << "invalid_call";
}
else
{
int size = slist[2].toInt();
res << QString::number(ft->RequestBlock(size));
}
}
else if (slist[1] == "WRITE_BLOCK")
{
if (slist.size() != 3)
{
LOG(VB_GENERAL, LOG_ERR, "Invalid QUERY_FILETRANSFER "
"WRITE_BLOCK call");
res << "ERROR" << "invalid_call";
}
else
{
int size = slist[2].toInt();
res << QString::number(ft->WriteBlock(size));
}
}
else if (slist[1] == "SEEK")
{
if (slist.size() != 5)
{
LOG(VB_GENERAL, LOG_ERR, "Invalid QUERY_FILETRANSFER SEEK call");
res << "ERROR" << "invalid_call";
}
else
{
long long pos = slist[2].toLongLong();
int whence = slist[3].toInt();
long long curpos = slist[4].toLongLong();
res << QString::number(ft->Seek(curpos, pos, whence));
}
}
else if (slist[1] == "SET_TIMEOUT")
{
if (slist.size() != 3)
{
LOG(VB_GENERAL, LOG_ERR, "Invalid QUERY_FILETRANSFER "
"SET_TIMEOUT call");
res << "ERROR" << "invalid_call";
}
else
{
bool fast = slist[2].toInt();
ft->SetTimeout(fast);
res << "ok";
}
}
else
{
LOG(VB_GENERAL, LOG_ERR, "Invalid QUERY_FILETRANSFER call");
res << "ERROR" << "invalid_call";
}
ft->DownRef();
socket->SendStringList(res);
return true;
}
bool FileServerHandler::HandleAnnounce(MythSocket *socket,
QStringList &commands, QStringList &slist)
{
if (commands[1] == "FileServer")
{
if (slist.size() >= 3)
{
SocketHandler *handler =
new SocketHandler(socket, m_parent, commands[2]);
handler->BlockShutdown(true);
handler->AllowStandardEvents(true);
handler->AllowSystemEvents(true);
QWriteLocker wlock(&m_fsLock);
m_fsMap.insert(commands[2], handler);
m_parent->AddSocketHandler(handler);
slist.clear();
slist << "OK";
handler->SendStringList(slist);
return true;
}
return false;
}
if (commands[1] != "FileTransfer")
return false;
if (slist.size() < 3)
return false;
if ((commands.size() < 3) || (commands.size() > 6))
return false;
FileTransfer *ft = NULL;
QString hostname = "";
QString filename = "";
bool writemode = false;
bool usereadahead = true;
int timeout_ms = 2000;
switch (commands.size())
{
case 6:
timeout_ms = commands[5].toInt();
case 5:
usereadahead = commands[4].toInt();
case 4:
writemode = commands[3].toInt();
default:
hostname = commands[2];
}
QStringList::const_iterator it = slist.begin();
QUrl qurl = *(++it);
QString wantgroup = *(++it);
QStringList checkfiles;
while (++it != slist.end())
checkfiles += *(it);
slist.clear();
LOG(VB_GENERAL, LOG_DEBUG, "FileServerHandler::HandleAnnounce");
LOG(VB_GENERAL, LOG_INFO, QString("adding: %1 as remote file transfer")
.arg(hostname));
if (writemode)
{
if (wantgroup.isEmpty())
wantgroup = "Default";
StorageGroup sgroup(wantgroup, gCoreContext->GetHostName(), false);
QString dir = sgroup.FindNextDirMostFree();
if (dir.isEmpty())
{
LOG(VB_GENERAL, LOG_ERR, "Unable to determine directory "
"to write to in FileTransfer write command");
slist << "ERROR" << "filetransfer_directory_not_found";
socket->writeStringList(slist);
return true;
}
QString basename = qurl.path();
if (basename.isEmpty())
{
LOG(VB_GENERAL, LOG_ERR, QString("FileTransfer write "
"filename is empty in url '%1'.")
.arg(qurl.toString()));
slist << "ERROR" << "filetransfer_filename_empty";
socket->writeStringList(slist);
return true;
}
if ((basename.contains("/../")) ||
(basename.startsWith("../")))
{
LOG(VB_GENERAL, LOG_ERR, QString("FileTransfer write "
"filename '%1' does not pass sanity checks.")
.arg(basename));
slist << "ERROR" << "filetransfer_filename_dangerous";
socket->writeStringList(slist);
return true;
}
filename = dir + "/" + basename;
}
else
filename = LocalFilePath(qurl, wantgroup);
QFileInfo finfo(filename);
if (finfo.isDir())
{
LOG(VB_GENERAL, LOG_ERR, QString("FileTransfer filename "
"'%1' is actually a directory, cannot transfer.")
.arg(filename));
slist << "ERROR" << "filetransfer_filename_is_a_directory";
socket->writeStringList(slist);
return true;
}
if (writemode)
{
QString dirPath = finfo.absolutePath();
QDir qdir(dirPath);
if (!qdir.exists())
{
if (!qdir.mkpath(dirPath))
{
LOG(VB_GENERAL, LOG_ERR, QString("FileTransfer "
"filename '%1' is in a subdirectory which does "
"not exist, but can not be created.")
.arg(filename));
slist << "ERROR" << "filetransfer_unable_to_create_subdirectory";
socket->writeStringList(slist);
return true;
}
}
ft = new FileTransfer(filename, socket, m_parent, writemode);
}
else
ft = new FileTransfer(filename, socket, m_parent, usereadahead, timeout_ms);
ft->BlockShutdown(true);
{
QWriteLocker wlock(&m_ftLock);
m_ftMap.insert(socket->socket(), ft);
}
slist << "OK"
<< QString::number(socket->socket())
<< QString::number(ft->GetFileSize());
if (checkfiles.size())
{
QFileInfo fi(filename);
QDir dir = fi.absoluteDir();
for (it = checkfiles.begin(); it != checkfiles.end(); ++it)
{
if (dir.exists(*it) &&
QFileInfo(dir, *it).size() >= kReadTestSize)
slist << *it;
}
}
socket->writeStringList(slist);
m_parent->AddSocketHandler(ft);
return true;
}
