static void progress(const char *acc, uint64_t sz,
uint64_t srcSz, uint64_t hSrc, char sfSrc, KTime_t date)
{
if (sz > 0) {
if (srcSz > 0) {
uint64_t p = 100 * sz / srcSz;
char sf = 'B';
uint64_t fr = 0;
uint64_t h = humanize(sz, &sf, &fr);
if (p > 0) {
if (sfSrc != 'B' && sf != 'B') {
if (fr == 0) {
if (date == 0) {
OUTMSG(("%s %,ld/%,ld %ld%c/%,ld%c %ld%% \r",
acc, sz, srcSz, h,sf,hSrc,sfSrc,p));
}
else {
OUTMSG(("%s %,ld/%,ld %ld%c/%,ld%c %ld%% %ld \r",
acc, sz, srcSz, h,sf,hSrc,sfSrc,p,
KTimeStamp() - date));
}
}
else {
OUTMSG(("%s %,ld/%,ld %ld.%03ld%c/%,ld%c %ld%% \r",
acc, sz, srcSz,h,fr,sf,hSrc,sfSrc,p));
}
}
else {
OUTMSG(("%s %,ld/%,ld %ld%% \r",
acc, sz, srcSz, p));
}
}
else {
if (sfSrc != 'B' && sf != 'B') {
if (fr == 0) {
OUTMSG((
"%s %,ld/%,ld %ld%c/%ld%c \r",
acc, sz,srcSz,h, sf,hSrc,sfSrc));
}
else {
OUTMSG((
"%s %,ld/%,ld %ld.%03ld%c/%ld%c \r",
acc, sz,srcSz,h, fr,sf,hSrc,sfSrc));
}
}
else {
OUTMSG(("%s %,ld/%,ld \r", acc, sz, srcSz));
}
}
}
else {
OUTMSG(("%s %,ld \r", acc, sz));
}
}
else {
OUTMSG((" \r%s\r", acc));
}
}
static rc_t fill_timestring( char * s, size_t size )
{
KTime tr;
rc_t rc;
KTimeLocal ( &tr, KTimeStamp() );
/*
rc = string_printf ( s, size, NULL,
"%.04u-%.02u-%.02u %.02u:%.02u:%.02u",
tr.year, tr.month + 1, tr.day, tr.hour, tr.minute, tr.second );
*/
rc = string_printf ( s, size, NULL, "%lT", &tr );
DISP_RC( rc, "fill_timestring:string_printf( date/time ) failed" );
return rc;
}
/* Make
*/
static
KThreadEvent * KThreadEventMake ( ctx_t ctx,
uint32_t lineno, xc_sev_t severity, xc_org_t origin,
xc_t xc, const char * msg, va_list args )
{
KThreadEvent * evt;
KTime_t ts = KTimeStamp ();
char * c;
size_t num_writ;
char msg_buffer [ 4096 ];
rc_t rc = string_vprintf ( msg_buffer, sizeof msg_buffer, & num_writ, msg, args );
if ( rc != 0 || num_writ >= sizeof msg_buffer )
string_printf ( msg_buffer, sizeof msg_buffer, & num_writ, "** BAD MESSAGE STRING **" );
if ( num_writ > 0 && msg_buffer [ num_writ - 1 ] == '.' )
msg_buffer [ -- num_writ ] = 0;
evt = malloc ( sizeof * evt + num_writ + 1 );
if ( evt == NULL )
{
/* ATTEMPT TO DUMP TO LOG */
KThreadEventDump ( ctx, ts, ctx -> loc, lineno, severity, origin, xc, msg_buffer, NULL );
exit ( -1 );
}
evt -> node = NULL;
evt -> next = NULL;
evt -> xc = ( const XCErr * ) xc;
evt -> timestamp = ts;
evt -> severity = severity;
evt -> origin = origin;
evt -> lineno = lineno;
c = ( char* ) ( evt + 1 );
memmove ( c, msg_buffer, num_writ + 1 );
StringInit ( & evt -> message, c, num_writ, string_len ( c, num_writ ) );
return evt;
}
rc_t run_ascp(const char *path, const char *key,
const char *src, const char *dest, const AscpOptions *opt)
{
const char *host = NULL;
const char *user = NULL;
const char *maxRate = NULL;
bool cache_key = false;
uint64_t heartbeat = 0;
const char *acc = NULL;
uint64_t srcSz = 0;
uint64_t id = 0;
TProgress *callback = NULL;
TQuitting *quitting = NULL;
rc_t rc = 0;
pid_t nPid = 0;
int pipeto[2]; /* pipe to feed the exec'ed program input */
int pipefrom[2]; /* pipe to get the exec'ed program output */
#define ARGV_SZ 64
char *argv[ARGV_SZ];
char extraOptions[4096] = "";
int i = 0;
int ret = 0;
i = 0;
if (opt != NULL) {
acc = opt->name;
cache_key = opt->cache_key;
callback = opt->callback;
heartbeat = opt->heartbeat;
host = opt->host;
id = opt->id;
quitting = opt->quitting;
srcSz = opt->src_size;
user = opt->user;
if (opt->ascp_options != NULL) {
size_t s = string_size(opt->ascp_options);
if (s >= sizeof extraOptions) {
LOGERR(klogErr,
RC(rcExe, rcProcess, rcCreating, rcBuffer, rcInsufficient),
"extra ascp options are ignored");
maxRate = opt->target_rate;
}
else {
string_copy
(extraOptions, sizeof extraOptions, opt->ascp_options, s);
}
}
else {
maxRate = opt->target_rate;
}
}
if (acc == NULL) {
acc = dest;
}
if (heartbeat > 0) {
heartbeat /= 1000;
if (heartbeat == 0) {
heartbeat = 1;
}
}
if (pipe(pipeto) != 0) {
perror("pipe() to");
rc = RC(rcExe, rcFileDesc, rcCreating, rcFileDesc, rcFailed);
LOGERR(klogErr, rc, "while pipe");
return rc;
}
if (pipe(pipefrom) != 0) {
perror("pipe() from");
rc = RC(rcExe, rcFileDesc, rcCreating, rcFileDesc, rcFailed);
LOGERR(klogErr, rc, "while pipe");
return rc;
}
argv[i++] = (char*)path;
argv[i++] = "-i";
argv[i++] = (char*)key;
argv[i++] = "-pQTk1";
if (maxRate != NULL && maxRate[0] != '\0') {
argv[i++] = "-l";
argv[i++] = (char*)maxRate;
}
if (user != NULL) {
argv[i++] = "--user";
argv[i++] = (char*)user;
}
if (host != NULL) {
argv[i++] = "--host";
argv[i++] = (char*)user;
}
if (extraOptions[0] != '\0') {
bool done = false;
char *c = extraOptions;
while (!done) {
while (true) {
if (*c == '\0') {
break;
}
else if (isspace(*c)) {
++c;
}
else {
break;
}
}
if (*c == '\0') {
break;
}
else {
argv[i++] = c;
}
while (true) {
if (*c == '\0') {
done = true;
break;
}
else if (isspace(*c)) {
*(c++) = '\0';
break;
}
++c;
}
if (i > ARGV_SZ - 4) {
LOGERR(klogErr,
RC(rcExe, rcProcess, rcCreating, rcBuffer, rcInsufficient),
"too mary extra ascp options - some of them are ignored");
break;
}
}
}
argv[i++] = (char*)src;
argv[i++] = (char*)dest;
argv[i++] = NULL;
logevp(path, argv);
if (quitting) {
rc = quitting();
}
if (rc != 0) {
return rc;
}
nPid = fork();
if (nPid < 0 ) {
perror("fork() 1");
rc = RC(rcExe, rcProcess, rcCreating, rcProcess, rcFailed);
LOGERR(klogErr, rc, "after fork");
return rc;
}
else if (nPid == 0) {
/* dup pipe read/write to stdin/stdout */
dup2(pipeto [0], STDIN_FILENO);
dup2(pipefrom[1], STDOUT_FILENO);
dup2(pipefrom[1], STDERR_FILENO);
close(pipeto[1]);
ret = execvp(path, argv);
STSMSG(STS_DBG, ("CHILD: Done %s %s %s = %d", path, src, dest, ret));
exit(EXIT_FAILURE);
}
else {
bool progressing = false;
bool writeFailed = false;
EAscpState state = eStart;
const char y[] = "y\n";
const char n[] = "n\n";
int status = 0;
int w = 0;
int fd = pipefrom[0];
const char *answer = n;
String line;
StringInit(&line, NULL, 0, 0);
if (cache_key) {
answer = y;
}
{
int flags = fcntl(fd, F_GETFL, 0);
fcntl(fd, F_SETFL, flags | O_NONBLOCK);
}
close(pipeto[0]);
close(pipefrom[1]);
assert(sizeof y == sizeof n);
{
int hang = 0;
uint64_t prev = 0;
KTime_t tPrev = 0;
char sfSrc = 'B';
uint64_t hSrc = humanize(srcSz, &sfSrc, NULL);
int sig = 0;
uint64_t i = 0;
KDirectory *dir = NULL;
rc_t rc = KDirectoryNativeDir(&dir);
DISP_RC(rc, "KDirectoryNativeDir");
if (rc != 0) {
return rc;
}
while (w == 0) {
bool quit = false;
w = waitpid(nPid, &status, WNOHANG);
if (w == 0) {
bool got = false;
rc_t rc = 0;
if (quitting) {
rc = quitting();
}
if (rc != 0 || quit) {
if (sig == 0) {
sig = SIGINT;
}
else if (sig >= SIGKILL) {
break;
}
else {
++sig;
}
if (progressing) {
OUTMSG(("\n"));
}
PLOGMSG(klogInfo, (klogInfo, "^C pressed: "
"Senging $(sgn) to ascp", "sgn=%s", sig));
kill(nPid, sig);
}
while (true) {
char buf[4096];
int s = read(fd, buf, sizeof buf);
if (s == 0) {
break;
}
else if (s < 0) {
if (errno != EAGAIN) {
if (progressing) {
OUTMSG(("\n"));
}
perror("read(child)");
}
break;
}
ascpParse(buf, s, dest, &state, &line);
switch (state) {
case eKeyEnd:
write(pipeto[1], answer, sizeof y - 1);
break;
case eWriteFailed:
writeFailed = true;
break;
default:
break;
}
got = true;
}
if (!got) {
sleep(1);
++i;
if ((heartbeat > 0 && i >= heartbeat) || (i > 99)) {
uint64_t size = 0;
rc_t rc = KDirectoryFileSize(dir, &size, "%s", dest);
if (rc != 0) {
size = 0;
}
else {
if (size != prev) {
prev = size;
tPrev = 0;
hang = 0;
}
else {
KTime_t date = 0;
rc_t rc = KDirectoryDate(dir, &date, "%s", dest);
if (rc == 0) {
tPrev = date;
if ((KTimeStamp() - date) > 60 * 99) {
/* no file update during 99' */
if (hang == 0) {
write(pipeto[1],
answer, sizeof y - 1);
++hang;
}
else if (hang < 9) {
++hang;
sig = 0;
}
else {
if (sig == 0) {
sig = SIGINT;
}
else {
++sig;
}
if (progressing) {
OUTMSG(("\n"));
}
if (sig > SIGKILL) {
rc = RC(rcExe,
rcProcess, rcExecuting,
rcProcess,rcDetached);
return rc;
}
PLOGMSG(klogInfo, (klogInfo,
"Senging $(sgn) to ascp",
"sgn=%s", sig));
kill(nPid, sig);
}
}
}
}
}
if (heartbeat > 0) {
if (callback) {
quit = !callback(id,
eAscpStateRunning, size, 0);
}
else {
progress(acc, size, srcSz, hSrc, sfSrc,
tPrev);
}
progressing = true;
}
i = 0;
}
}
}
}
RELEASE(KDirectory, dir);
}
if (progressing) {
OUTMSG(("\n"));
}
while (1) {
char buf[4096];
int s = read(fd, buf, sizeof buf);
if (s == 0) {
break;
}
else if (s < 0) {
if (errno != EAGAIN) {
perror("read(child)");
break;
}
continue;
}
ascpParse(buf, s, dest, &state, &line);
if (state == eWriteFailed) {
writeFailed = true;
}
}
STSMSG( STS_DBG, ("ascp exited with pid=%d status=%d", w, status));
if (WIFEXITED(status)) {
STSMSG(STS_DBG, ("ascp exited with exit status %d",
WEXITSTATUS(status)));
}
else {
STSMSG(STS_DBG, ("ascp has not terminated correctly"));
}
if (w == -1) {
perror("waitpid");
rc = RC(rcExe, rcProcess, rcWaiting, rcProcess, rcFailed);
LOGERR(klogErr, rc, "after waitpid");
exit(EXIT_FAILURE);
}
if (WIFEXITED(status)) {
if (WEXITSTATUS(status) == 0) {
STSMSG(STS_DBG, ("ascp succeed"));
if (callback) {
callback(id, eAscpStateExitSuccess, 0, 0);
}
} else if (writeFailed) {
rc = RC(rcExe, rcProcess, rcExecuting, rcMemory, rcExhausted);
if (callback) {
callback(id, eAscpStateExitWriteFailure, 0, 0);
}
}
else {
if (rc == 0) {
rc = RC(rcExe, rcProcess, rcWaiting, rcProcess, rcFailed);
}
PLOGERR(klogErr, (klogErr, rc,
"ascp failed with $(ret)", "ret=%d", WEXITSTATUS(status)));
if (callback) {
callback(id, eAscpStateExitFailure, 0, 0);
}
}
} else if (WIFSIGNALED(status)) {
if (rc == 0) {
if (quitting) {
rc = quitting();
if (rc == 0) {
rc = RC(rcExe,
rcProcess, rcWaiting, rcProcess, rcFailed);
}
}
}
if (rc != SILENT_RC(rcExe, rcProcess, rcExecuting,
rcProcess, rcCanceled))
{
PLOGERR(klogErr, (klogErr, rc, "ascp killed by signal $(sig)",
"sig=%d", WTERMSIG(status)));
if (callback) {
callback(id, eAscpStateExitFailure, 0, 0);
}
}
}
}
return rc;
}
static rc_t write_header( cg_dump_opts * opts, struct sg_lookup * lookup, lane * l )
{
char buffer[ 1024 ];
size_t num_writ_buf;
sg * entry = NULL;
rc_t rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#GENERATED_BY\t%s\n", UsageDefaultName );
if ( rc != 0 )
{
(void)LOGERR( klogErr, rc, "cannot generate GENERATED_BY for header" );
}
else
rc = write_header_line( l, buffer, num_writ_buf );
if ( rc == 0 )
{
KTime now;
KTimeLocal ( &now, KTimeStamp () );
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#GENERATED_AT\t%lT\n", &now );
if ( rc != 0 )
{
(void)LOGERR( klogErr, rc, "cannot generate GENERATED_AT for header" );
}
else
rc = write_header_line( l, buffer, num_writ_buf );
}
if ( rc == 0 )
{
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#SOFTWARE_VERSION\t%.3V\n", KAppVersion() );
if ( rc != 0 )
{
(void)LOGERR( klogErr, rc, "cannot generate SOFTWARE_VERSION for header" );
}
else
rc = write_header_line( l, buffer, num_writ_buf );
}
if ( rc == 0 )
{
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#FORMAT_VERSION\t2.0\n" );
if ( rc != 0 )
{
(void)LOGERR( klogErr, rc, "cannot generate FORMAT_VERSION for header" );
}
else
rc = write_header_line( l, buffer, num_writ_buf );
}
if ( rc == 0 )
{
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#TYPE\tSAM_READS\n" );
if ( rc != 0 )
{
(void)LOGERR( klogErr, rc, "cannot generate SAM_READS for header" );
}
else
rc = write_header_line( l, buffer, num_writ_buf );
}
if ( rc == 0 )
{
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#SLIDE\t%.*s\n", ( l->name->len - 4 ), l->name->addr );
if ( rc != 0 )
{
(void)LOGERR( klogErr, rc, "cannot generate SLIDE for header" );
}
else
rc = write_header_line( l, buffer, num_writ_buf );
}
if ( rc == 0 )
{
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#LANE\t%.*s\n", 3, l->name->addr + ( ( l->name->len - 3 ) ) );
if ( rc != 0 )
{
(void)LOGERR( klogErr, rc, "cannot generate LANE for header" );
}
else
rc = write_header_line( l, buffer, num_writ_buf );
}
if ( rc == 0 )
{
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#BATCH_FILE_NUMBER\t%d\n", l->chunk );
if ( rc != 0 )
{
(void)LOGERR( klogErr, rc, "cannot generate BATCH_FILE_NUMBER for header" );
}
else
rc = write_header_line( l, buffer, num_writ_buf );
}
if ( rc == 0 )
{
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#LANE\t%S\n", l->name );
if ( rc != 0 )
{
(void)LOGERR( klogErr, rc, "cannot generate LANE(2) for header" );
}
else
rc = write_header_line( l, buffer, num_writ_buf );
}
if ( rc == 0 )
{
perform_sg_lookup( lookup, l->name, &entry );
if ( entry != NULL )
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#LIBRARY\t%S\n", &entry->lib );
else
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#LIBRARY\t%s\n", opts->lib ); /* opts->lib */
if ( rc != 0 )
{
(void)LOGERR( klogErr, rc, "cannot generate LIBRARY for header" );
}
else
rc = write_header_line( l, buffer, num_writ_buf );
}
if ( rc == 0 )
{
if ( entry != NULL )
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#SAMPLE\t%S\n", &entry->sample );
else
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#SAMPLE\t%s\n", opts->sample ); /* opts->sample */
if ( rc != 0 )
{
(void)LOGERR( klogErr, rc, "cannot generate SAMPLE for header" );
}
else
rc = write_header_line( l, buffer, num_writ_buf );
}
if ( rc == 0 )
{
if ( entry != NULL )
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#FIELD_SIZE\t%S\n", &entry->field_size );
else
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "#FIELD_SIZE\t460800\n" );
if ( rc != 0 )
{
(void)LOGERR( klogErr, rc, "cannot generate FIELD_SIZE for header" );
}
else
rc = write_header_line( l, buffer, num_writ_buf );
}
if ( rc == 0 )
{
rc = string_printf ( buffer, sizeof buffer, &num_writ_buf, "\n>readOffset\tside\tbases\tscores\n" );
if ( rc != 0 )
{
(void)LOGERR( klogErr, rc, "cannot generate columns for header" );
}
else
rc = write_header_line( l, buffer, num_writ_buf );
}
return rc;
}