static
int64_t CC sort_cmp (const void ** l, const void ** r, void * data)
{
/* KDirectory * d; */
uint64_t lz, rz;
rc_t rc;
/* d = data; */
/* lz = l; */
/* rz = r; */
rc = KDirectoryFileSize (data, &lz, *l);
if (rc == 0)
{
rc = KDirectoryFileSize (data, &rz, *r);
if (rc == 0)
{
int64_t zdiff;
zdiff = lz - rz;
if (zdiff != 0)
return (zdiff < 0) ? -1 : 1;
else
{
size_t lsz = string_size (*l);
size_t rsz = string_size (*r);
return string_cmp (*l, lsz, *r, rsz, lsz+1);
}
}
}
return 0; /* dunno just leave it... */
}
static rc_t CC visitor(const KDirectory* dir,
uint32_t type, const char* name, void* data)
{
rc_t rc = 0;
Total* total = (Total*) data;
if (type & kptAlias)
{ return rc; }
assert(total);
switch (type) {
case kptFile: {
uint64_t size = 0;
rc = KDirectoryFileSize(dir, &size, "%s", name);
if (rc == 0) {
total->sz += size;
}
++total->files;
break;
}
case kptDir:
rc = KDirectoryVisit(dir, false, visitor, total, "%s", name);
break;
default:
rc = RC(rcVDB, rcDirectory, rcVisiting, rcType, rcUnexpected);
break;
}
return rc;
}
static
rc_t CC KLoaderFile_Size(const KLoaderFile *self, uint64_t *size)
{
if( self && self->kfile ) {
return KFileSize(self->kfile, size);
}
return KDirectoryFileSize(self ? self->dir : NULL, size, "%s", self->realname);
}
static rc_t on_clear_path( visit_ctx * obj )
{
rc_t rc = 0;
clear_data * data = obj->data;
/* the caller is written to first hit the files/subdirs, then the directory that contains them */
if ( obj->path_type == kptDir )
{
if ( obj->options->remove_dirs )
{
rc = KDirectoryRemove ( obj->dir, false, "%s", obj->path );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
"KDirectoryRemove( $(path) ) failed in $(func)", "path=%s,func=%s", obj->path, __func__ ) );
}
else
data->removed_directories++;
}
}
else if ( obj->path_type == kptFile )
{
uint64_t file_size;
rc = KDirectoryFileSize ( obj->dir, &file_size, "%s", obj->path );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
"KDirectoryFileSize( $(path) ) failed in $(func)", "path=%s,func=%s", obj->path, __func__ ) );
}
else
{
rc = KDirectoryRemove ( obj->dir, false, "%s", obj->path );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
"KDirectoryRemove( $(path) ) failed in $(func)", "path=%s,func=%s", obj->path, __func__ ) );
}
else
{
if ( obj->options->detailed )
rc = KOutMsg( "FILE: '%s' removed (%,u bytes)\n", obj->path, file_size );
data->removed_files++;
data->removed_size += file_size;
if ( obj->options->max_remove > 0 && data->removed_size >= obj->options->max_remove )
{
obj->terminate = true;
KOutMsg( "the maximum of %,lu bytes to be removed is reached now\n", obj->options->max_remove );
}
}
}
}
return rc;
}
static rc_t on_report_full_file( visit_ctx * obj )
{
rc_t rc = 0;
uint64_t file_size = 0;
report_data * data = obj->data;
data->full_count++;
rc = KDirectoryFileSize ( obj->dir, &file_size, "%s", obj->path );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
"KDirectoryFileSize( $(path) ) failed in $(func)", "path=%s,func=%s", obj->path, __func__ ) );
}
else
{
data->file_size += file_size;
data->used_file_size += file_size;
if ( obj->options->detailed )
rc = KOutMsg( "%s complete file of %,u bytes\n", obj->path, file_size );
}
return rc;
}
static rc_t CC ReportObj(const ReportFuncs *f, uint32_t indent,
const char *object, bool *wasDbOrTableSet)
{
Report* self = NULL;
const char* fullpath = NULL;
const KDatabase* kdb = NULL;
const KTable* ktbl = NULL;
const VDatabase* db = NULL;
KPathType type = kptNotFound;
KPathType file_type = kptNotFound;
bool alias = false;
uint64_t size = 0;
bool size_unknown = true;
rc_t rc = ReportGet(&self);
assert(self);
if (wasDbOrTableSet != NULL) {
*wasDbOrTableSet = self->db != NULL || self->table != NULL;
return 0;
}
if (self->db != NULL) {
type = kptDatabase;
db = self->db;
}
else if (self->table != NULL) {
rc_t rc2 = VTableOpenParentRead(self->table, &db);
if (rc2)
{
if (rc == 0)
{
rc = rc2;
}
}
else if (!db)
{
type = kptTable;
rc2 = VTableGetKTableRead(self->table, &ktbl);
if (rc2)
{
if (rc == 0)
{
rc = rc2;
}
}
else
{
rc2 = KTableGetPath(ktbl, &fullpath);
}
}
}
if (db) {
rc_t rc2 = VDatabaseOpenKDatabaseRead(db, &kdb);
type = kptDatabase;
if (rc2) {
if (rc == 0)
{ rc = rc2; }
}
else {
rc2 = KDatabaseGetPath(kdb, &fullpath);
if (rc2) {
if (rc == 0)
{ rc = rc2; }
}
}
}
if (fullpath) {
KDirectory* dir = NULL;
rc_t rc2 = KDirectoryNativeDir(&dir);
if (rc2) {
if (rc == 0)
{ rc = rc2; }
}
else {
file_type = KDirectoryPathType(dir, "%s", fullpath);
alias = file_type & kptAlias;
file_type &= ~kptAlias;
if (file_type == kptFile) {
rc2 = KDirectoryFileSize(dir, &size, "%s", fullpath);
if (rc2) {
if (rc == 0)
{ rc = rc2; }
}
else { size_unknown = false; }
}
}
RELEASE(KDirectory, dir);
}
if (object || type != kptNotFound) {
const char* path
= fullpath ? fullpath : object ? object : "not set";
const char* stype = type == kptTable ? "table" :
type == kptDatabase ? "database" : "unknown";
const char* sfile_type = file_type == kptFile ? "archive" :
file_type == kptDir ? "dir" : "unexpected";
if (fullpath && !size_unknown) {
if (alias)
{ OBJ_P_S_A(indent, path, stype, sfile_type, size); }
else
{ OBJ_P_S (indent, path, stype, sfile_type, size); }
}
else if (fullpath && size_unknown) {
if (alias)
{ OBJ_P_A (indent, path, stype, sfile_type); }
else
{ OBJ_P (indent, path, stype, sfile_type); }
}
else
{ OBJ (indent, path, stype); }
if (!db)
{ db = self->db; }
if (db) {
rc_t rc2 = ReportDepend(f, indent + 1, db);
if (rc == 0)
{ rc = rc2; }
}
if (file_type == kptDir) {
rc_t rc2 = ReportDir(f, indent + 1, ktbl);
if (rc == 0)
{ rc = rc2; }
}
reportClose(indent, "Object");
}
if (db != self->db)
{ RELEASE(VDatabase, db); }
RELEASE(KTable, ktbl);
RELEASE(KDatabase, kdb);
return rc;
}
static rc_t VDBDependenciesReportDepend1(const VDBDependencies *self,
const ReportFuncs *f,
uint32_t count, uint32_t indent, bool toreport, uint32_t *missing)
{
KDirectory *dir = NULL;
rc_t rc = 0;
uint32_t i = ~0;
assert(missing);
*missing = 0;
for (i = 0; i < count; ++i) {
bool isMissing = false;
bool local = false;
const char* seq_id = "";
const char* path = NULL;
rc = VDBDependenciesSeqId(self, &seq_id, i);
if (rc != 0 && toreport)
reportErrorStrInt(indent, rc, "VDBDependenciesSeqId",
"origin", "VDatabaseListDependencies", "idx", i);
if (rc == 0) {
rc = VDBDependenciesLocal(self, &local, i);
if (rc != 0 && toreport) {
reportErrorStrInt(indent, rc,
"VDBDependenciesLocal",
"origin", "VDatabaseListDependencies", "idx", i);
}
}
if (rc == 0) {
rc = VDBDependenciesPath(self, &path, i);
if (rc != 0 && toreport) {
reportErrorStrInt(indent, rc, "VDBDependenciesPath",
"origin", "VDatabaseListDependencies", "idx", i);
}
}
if (rc == 0) {
if (!local && (path == NULL || path[0] == '\0')) {
isMissing = true;
++*missing;
}
if (toreport) {
bool reported = false;
if (!isMissing && !local) {
rc_t rc = 0;
bool readable = false;
uint64_t size = ~0;
KTime_t date = 0;
bool ready = false;
if (dir == NULL)
{ rc = KDirectoryNativeDir(&dir); }
assert(!local && path && path[0]);
if (dir != NULL) {
rc = KDirectoryFileSize(dir, &size, "%s", path);
if (rc == 0)
{ rc = KDirectoryDate(dir, &date, "%s", path); }
if (rc == 0) {
const KFile* f = NULL;
rc = KDirectoryOpenFileRead(dir, &f, "%s", path);
if (rc == 0) {
char buffer[1024];
size_t num_read = 0;
size_t bsize =
size > sizeof buffer ? sizeof buffer : size;
rc = KFileReadAll(f,
0, buffer, bsize, &num_read);
if (rc == 0 && num_read != bsize) {
rc = RC(rcVDB, rcFile, rcReading,
rcBuffer, rcExcessive);
}
if (rc == 0)
{ readable = true; }
}
KFileRelease(f);
}
if (rc == 0)
{ ready = true; }
}
if (ready) {
KTime kt;
memset(&kt, 0, sizeof kt);
KTimeLocal(&kt, date);
report(indent, "Dependency", 7,
"index", 'd', i,
"seq_id", 's', seq_id,
"local", 's', local ? "true" : "false",
"path", 's', path,
"size", 'u', size,
"date", 'T', &kt,
"readable", 's', readable ? "true" : "false");
reported = true;
}
else {
report(indent, "Dependency", 5,
"index", 'd', i,
"seq_id", 's', seq_id,
"local", 's', local ? "true" : "false",
"path", 's', path,
"error", 'R', rc);
reported = true;
}
}
if (!reported) {
report(indent, "Dependency", 4,
"index", 'd', i,
"seq_id", 's', seq_id,
"local", 's', local ? "true" : "false",
"path", 's', path == NULL ? "" : path);
}
}
}
if (rc != 0 && !toreport)
{ break; }
}
RELEASE(KDirectory, dir);
return rc;
}
LIB_EXPORT rc_t CC aspera_get(
const char *ascp_bin, const char *private_file, const char *aSrc,
const char *dest, AscpOptions *opt)
{
char path[PATH_MAX] = "";
AscpOptions dummy;
bool status = false;
int64_t prev = -1;
int attempt = 0;
KDirectory *dir = NULL;
TQuitting *quitting = NULL;
const char *src = aSrc;
rc_t rc = KDirectoryNativeDir(&dir);
if (rc != 0) {
return rc;
}
if (ascp_bin == NULL || private_file == NULL ||
src == NULL || dest == NULL)
{
return RC(rcNS, rcFile, rcCopying, rcParam, rcNull);
}
if (opt == NULL) {
memset(&dummy, 0, sizeof dummy);
opt = &dummy;
}
if (opt->ascp_options == NULL && opt->target_rate[0] == '\0') {
KConfig *cfg = NULL;
rc_t rc = KConfigMake(&cfg, NULL);
DISP_RC(rc, "cannot KConfigMake");
if (rc == 0) {
rc = _KConfigGetAscpRate(cfg,
opt->target_rate, sizeof opt->target_rate);
DISP_RC(rc, "cannot get aspera max rate");
}
RELEASE(KConfig, cfg);
}
sStatus = status = opt->status;
quitting = opt->quitting;
{
/* remove trailing #... or ?... from src path:
it could come from Revolver */
size_t s = string_size(aSrc);
const char *n = string_chr(aSrc, s, '#');
const char *q = string_chr(aSrc, s, '?');
if (q != NULL && (n == NULL || q < n)) {
n = q;
}
if (n != NULL) {
if (n - aSrc + 1 > sizeof path) {
return RC(rcNS, rcFile, rcCopying, rcBuffer, rcInsufficient);
}
else {
#if _DEBUGGING
size_t s =
#endif
string_copy(path, sizeof path, aSrc, n - aSrc);
assert(s <= sizeof path);
src = path;
}
}
}
while (true) {
rc = run_ascp(ascp_bin, private_file, src, dest, opt);
if (rc == 0) {
if (status) {
STSMSG(STS_DBG, ("ascp finished with success"));
}
break;
}
else if (rc == SILENT_RC(rcExe,
rcProcess, rcExecuting, rcMemory, rcExhausted))
{
if (status) {
STSMSG(STS_DBG, ("ascp failed: %R", rc));
}
break;
}
else {
rc_t rc2 = 0;
uint64_t size = 0;
if (quitting != NULL) {
rc2 = quitting();
if (rc2 != 0) {
break;
}
}
if (status) {
STSMSG(STS_DBG, ("ascp failed: %R", rc));
}
rc2 = KDirectoryFileSize(dir, &size, "%s", dest);
if (rc2 ==
SILENT_RC(rcFS, rcDirectory, rcAccessing, rcPath, rcNotFound))
{
if (prev < 0) {
if (status) {
STSMSG(0, ("fasp download failed. "
"File not found. Retrying..."));
}
prev = 0;
}
else {
if (status) {
STSMSG(0, ("fasp download failed. File not found."));
}
break;
}
}
else if (rc2 != 0 || (int64_t)size < prev) {
if (status) {
STSMSG(0, ("fasp download failed. KDirectoryFileSize "
"after ascp run: rc = %ld, size = %ld", rc, size));
}
break;
}
else if ((int64_t)size > prev) {
if (status) {
STSMSG(STS_INFO, (" fasp download failed. %ld bytes "
"received so far. Retrying...", size));
}
attempt = 0;
prev = size;
}
else {
if (attempt++ > 3) {
break;
}
if (status) {
STSMSG(STS_INFO, (" fasp download failed. %ld bytes "
"received so far. Retrying %d...", size, attempt));
}
}
}
}
RELEASE(KDirectory, dir);
return rc;
}
static
rc_t CC
_SFTestMake ( const struct _SFTest ** Test, const char * Path )
{
rc_t RCt;
struct _SFTest * Ret;
struct KDirectory * Dir;
uint32_t PathType;
uint64_t Size;
RCt = 0;
Ret = NULL;
Dir = NULL;
PathType = kptNotFound;
Size = 0;
if ( Test != NULL ) {
* Test = NULL;
}
if ( Test == NULL || Path == NULL ) {
return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcNull );
}
/* First we shoud check that file exist
*/
RCt = KDirectoryNativeDir ( & Dir );
if ( RCt == 0 ) {
PathType = KDirectoryPathType ( Dir, Path );
if ( PathType == kptFile ) {
RCt = KDirectoryFileSize ( Dir, & Size, Path );
if ( Size == 0 ) {
return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcInvalid );
}
}
else {
return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcInvalid );
}
KDirectoryRelease ( Dir );
}
if ( RCt == 0 ) {
Ret = calloc ( 1, sizeof ( struct _SFTest ) );
if ( Ret == NULL ) {
RCt = RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcExhausted );
}
else {
Ret -> path = string_dup_measure ( Path, NULL );
if ( Ret -> path == NULL ) {
RCt = RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcExhausted );
}
else {
Ret -> size = Size;
* Test = Ret;
}
}
}
if ( RCt != 0 ) {
* Test = NULL;
_SFTestDispose ( Ret );
}
return RCt;
} /* _SFTestMake () */
static
rc_t CC
_ReadAll ( const char * Path, char ** Buf, size_t * BufSize )
{
rc_t RCt;
const struct KFile * File;
struct KDirectory * Dir;
char * RetBuf;
size_t Size, NumRead;
RCt = 0;
File = NULL;
Dir = NULL;
RetBuf = NULL;
Size = 0;
NumRead = 0;
RCt = KDirectoryNativeDir ( & Dir );
if ( RCt == 0 ) {
RCt = KDirectoryFileSize ( Dir, & Size, Path );
if ( RCt == 0 ) {
if ( Size == 0 ) {
return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcInvalid );
}
else {
RetBuf = calloc ( Size, sizeof ( char ) );
if ( RetBuf == NULL ) {
return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcExhausted );
}
else {
RCt = KDirectoryOpenFileRead ( Dir, & File, Path );
if ( RCt == 0 ) {
RCt = KFileReadAll (
File,
0,
RetBuf,
Size,
& NumRead
);
if ( RCt == 0 ) {
* Buf = RetBuf;
* BufSize = Size;
}
}
}
}
}
KDirectoryRelease ( Dir );
}
if ( RCt != 0 ) {
Buf = NULL;
BufSize = 0;
if ( RetBuf != NULL ) {
free ( RetBuf );
}
}
return RCt;
} /* _ReadAll () */
rc_t run (const char * table_path, uint64_t N )
{
static const char *colInf[] = {
"C1: Same value, same length",
"C2: Var. value, same length",
"C3: Var. value, var. legnth",
"C4: Same value except I row",
"C5: Same value except L row" };
rc_t rc;
uint64_t row = 0;
VDBManager *mgr = NULL;
VSchema *schema = NULL;
VTable *table = NULL;
VCursor *cursor;
uint32_t idx[COLUMNS];
uint64_t total[COLUMNS];
int i = 0, j = 0, prev = 0;
char *buffer[BUFFERS];
/* Initialize arrays */
memset(&idx, 0, sizeof idx);
memset(&total, 0, sizeof total);
for (i = 0; i < BUFFERS; ++i) {
char c;
size_t sz = ROWLEN + 1;
if (i == (BUFFERS - 1))
sz += ROWLEN;
buffer[i] = malloc(sz);
for (j = 0, c = 0; j < sz - 1; ++j, ++c) {
if (c >= ROWLEN)
c -= ROWLEN;
buffer[i][j] = '0' + c;
}
buffer[i][j] = '\0';
}
/* Create manager */
rc = VDBManagerMakeUpdate(&mgr, NULL);
if (rc != 0) {
LOGERR(klogInt, rc, "failed to open vdb library");
}
/* Initialize schema */
if (rc == 0) {
rc = VDBManagerMakeSchema(mgr, &schema);
if (rc != 0)
LOGERR(klogInt, rc, "failed to create empty schema");
}
if (rc == 0) {
char text[512] = "table Table #1 {\n";
char col [128];
for (i = 1; i <= COLUMNS; ++i) {
sprintf(col,
" column ascii C%d = .C%d; physical ascii .C%d = C%d;\n",
i, i, i, i);
strcat(text, col);
}
strcat(text, "};");
STSMSG(1,("Parsing schema:\n%s", text));
rc = VSchemaParseText(schema, "Schema", text, strlen(text));
if (rc != 0)
LOGERR(klogInt, rc, "failed to parse schema");
}
/* Create table */
if (rc == 0) {
STSMSG(1,("Creating %s", tablePath));
rc = VDBManagerCreateTable(mgr, &table, schema, "Table", kcmInit,
tablePath);
if (rc != 0)
LOGERR(klogInt, rc, "failed to create table");
}
/* Initialize cursor */
if (rc == 0) {
rc = VTableCreateCursorWrite(table, &cursor, kcmInsert);
if (rc != 0)
LOGERR(klogInt, rc, "failed to create cursor");
}
for (i = 0; rc == 0 && i < COLUMNS; ++i) {
char col[3];
sprintf(col, "C%d", i + 1);
STSMSG(2,("Adding column %s to cursor", col));
rc = VCursorAddColumn(cursor, &idx[i], col);
if (rc != 0)
PLOGERR(klogInt, (klogInt, rc,
"failed to add $(c) to cursor", "c=%s", col));
}
if (rc == 0) {
rc = VCursorOpen(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to open cursor");
}
/* Write data */
for (row = 0; row < N && rc == 0; ++row) {
int max = 2 * ROWLEN - 1;
int sz = 0;
if ((row % 2) == 0) {
int min = 1;
sz = min + (int) (max * (rand() / (RAND_MAX + 1.0)));
prev = sz;
buffer[1][0] = '1';
}
else {
sz = max + 1 - prev;
buffer[1][0] = '2';
}
rc = Quitting();
if (rc == 0) {
KStsLevel lvl = 2;
if (row > 0 && ((row % ROWS) == 0)) {
lvl = 1;
}
STSMSG (lvl, ("Writing row %ji / %ji",
row + 1, N));
rc = VCursorOpenRow(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to open row");
}
for (j = 0; j < COLUMNS && rc == 0; ++j) {
uint32_t count = 0;
int buf = j;
switch (j) {
case 0:
case 1:
count = strlen(buffer[j]);
break;
case 2:
count = sz;
break;
case 3:
buf = 0;
if (row == 0)
buf = 1;
count = strlen(buffer[buf]);
break;
case 4:
buf = 0;
if (row == (N - 1))
buf = 1;
count = strlen(buffer[buf]);
break;
default:
assert(0);
break;
}
STSMSG (3, ("Row %ji/Col.%d: %sd %.*s\n",
row + 1, j + 1, count, count, buffer[buf]));
rc = VCursorWrite
(cursor, idx[j], 8, buffer[buf], 0, count);
if (rc != 0)
PLOGERR(klogInt, (klogInt, rc, "failed to write row[$i]", "i=%d", j + 1));
total[j] += count;
}
if (rc == 0) {
rc = VCursorCommitRow(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to commit row");
}
if (rc == 0) {
rc = VCursorCloseRow(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to close row");
}
}
if (rc == 0) {
STSMSG (1, ("Commiting cursor\n"));
rc = VCursorCommit(cursor);
if (rc != 0)
LOGERR(klogInt, rc, "failed to commit cursor");
}
/* Cleanup */
VCursorRelease(cursor);
VTableRelease(table);
VSchemaRelease(schema);
VDBManagerRelease(mgr);
for (i = 0; i < BUFFERS; ++i) {
free(buffer[i]);
}
/* Log */
if (rc == 0) {
PLOGMSG(klogInfo, (klogInfo, "$(t)", "t=%s", tablePath));
PLOGMSG(klogInfo,(klogInfo,
"$(n)($(N)) rows written - $(b) bytes per row",
PLOG_I64(n) "," PLOG_X64(N) ",b=%d", N, N, ROWLEN));
for (i = 0; i < COLUMNS; ++i) {
PLOGMSG(klogInfo,(klogInfo,
"$(i): $(n)($(N)) bytes",
"i=%s," PLOG_I64(n) "," PLOG_X64(N),
colInf[i], total[i], total[i]));
}
}
if (rc == 0) {
KDirectory *dir = NULL;
uint64_t sizes[COLUMNS];
memset(&sizes, 0, sizeof sizes);
rc = KDirectoryNativeDir(&dir);
if (rc != 0)
LOGERR(klogInt, rc, "failed to KDirectoryNativeDir");
else {
for (i = 1; i <= COLUMNS; ++i) {
uint64_t size = 0;
#define FORMAT "%s/col/%s/data"
#define KFORMAT "$(d)/col/$(n)/data", "d=%s,n=%s"
#define STATUS(action) (action FORMAT, tablePath, name)
char name[3];
sprintf(name, "C%d", i);
STSMSG (1, STATUS("checking "));
rc = KDirectoryFileSize(dir, &size, FORMAT, tablePath, name);
if (rc != 0) {
if (GetRCState(rc) == rcNotFound) {
STSMSG (2, STATUS("not found "));
rc = 0;
}
else
PLOGERR(klogInt, (klogInt, rc,
"failed to check " KFORMAT, tablePath, name));
}
else {
STSMSG (2, STATUS("found "));
}
PLOGMSG(klogInfo, (klogInfo, "Size of $(d)/col/$(n)/data = $(s)",
"d=%s,n=%s," PLOG_I64(s), tablePath, name, size));
sizes[i - 1] = size;
}
}
KDirectoryRelease(dir);
if (rc == 0) {
puts("");
KOutMsg("%ld rows, %d bytes per row:\n", N, ROWLEN);
for (i = 0; i < COLUMNS; ++i) {
puts(colInf[i]);
}
puts("");
for (i = 0; i < COLUMNS; ++i) {
int64_t over = sizes[i] - total[i];
KOutMsg("C%d: %9ld bytes written; "
"%9ld in 'data'", i + 1, total[i], sizes[i]);
if (over > 0) {
double p = 100.0 * over / sizes[i];
printf(": %7ld extra bytes (%.4f%%)\n", over, p);
}
else {
puts("");
}
}
}
}
return rc;
}
rc_t run_ascp(const char *path, const char *key,
const char *src, const char *dest, const AscpOptions *opt)
{
const char *host = NULL;
const char *user = NULL;
const char *maxRate = NULL;
bool cache_key = false;
uint64_t heartbeat = 0;
const char *acc = NULL;
uint64_t srcSz = 0;
uint64_t id = 0;
TProgress *callback = NULL;
TQuitting *quitting = NULL;
rc_t rc = 0;
pid_t nPid = 0;
int pipeto[2]; /* pipe to feed the exec'ed program input */
int pipefrom[2]; /* pipe to get the exec'ed program output */
#define ARGV_SZ 64
char *argv[ARGV_SZ];
char extraOptions[4096] = "";
int i = 0;
int ret = 0;
i = 0;
if (opt != NULL) {
acc = opt->name;
cache_key = opt->cache_key;
callback = opt->callback;
heartbeat = opt->heartbeat;
host = opt->host;
id = opt->id;
quitting = opt->quitting;
srcSz = opt->src_size;
user = opt->user;
if (opt->ascp_options != NULL) {
size_t s = string_size(opt->ascp_options);
if (s >= sizeof extraOptions) {
LOGERR(klogErr,
RC(rcExe, rcProcess, rcCreating, rcBuffer, rcInsufficient),
"extra ascp options are ignored");
maxRate = opt->target_rate;
}
else {
string_copy
(extraOptions, sizeof extraOptions, opt->ascp_options, s);
}
}
else {
maxRate = opt->target_rate;
}
}
if (acc == NULL) {
acc = dest;
}
if (heartbeat > 0) {
heartbeat /= 1000;
if (heartbeat == 0) {
heartbeat = 1;
}
}
if (pipe(pipeto) != 0) {
perror("pipe() to");
rc = RC(rcExe, rcFileDesc, rcCreating, rcFileDesc, rcFailed);
LOGERR(klogErr, rc, "while pipe");
return rc;
}
if (pipe(pipefrom) != 0) {
perror("pipe() from");
rc = RC(rcExe, rcFileDesc, rcCreating, rcFileDesc, rcFailed);
LOGERR(klogErr, rc, "while pipe");
return rc;
}
argv[i++] = (char*)path;
argv[i++] = "-i";
argv[i++] = (char*)key;
argv[i++] = "-pQTk1";
if (maxRate != NULL && maxRate[0] != '\0') {
argv[i++] = "-l";
argv[i++] = (char*)maxRate;
}
if (user != NULL) {
argv[i++] = "--user";
argv[i++] = (char*)user;
}
if (host != NULL) {
argv[i++] = "--host";
argv[i++] = (char*)user;
}
if (extraOptions[0] != '\0') {
bool done = false;
char *c = extraOptions;
while (!done) {
while (true) {
if (*c == '\0') {
break;
}
else if (isspace(*c)) {
++c;
}
else {
break;
}
}
if (*c == '\0') {
break;
}
else {
argv[i++] = c;
}
while (true) {
if (*c == '\0') {
done = true;
break;
}
else if (isspace(*c)) {
*(c++) = '\0';
break;
}
++c;
}
if (i > ARGV_SZ - 4) {
LOGERR(klogErr,
RC(rcExe, rcProcess, rcCreating, rcBuffer, rcInsufficient),
"too mary extra ascp options - some of them are ignored");
break;
}
}
}
argv[i++] = (char*)src;
argv[i++] = (char*)dest;
argv[i++] = NULL;
logevp(path, argv);
if (quitting) {
rc = quitting();
}
if (rc != 0) {
return rc;
}
nPid = fork();
if (nPid < 0 ) {
perror("fork() 1");
rc = RC(rcExe, rcProcess, rcCreating, rcProcess, rcFailed);
LOGERR(klogErr, rc, "after fork");
return rc;
}
else if (nPid == 0) {
/* dup pipe read/write to stdin/stdout */
dup2(pipeto [0], STDIN_FILENO);
dup2(pipefrom[1], STDOUT_FILENO);
dup2(pipefrom[1], STDERR_FILENO);
close(pipeto[1]);
ret = execvp(path, argv);
STSMSG(STS_DBG, ("CHILD: Done %s %s %s = %d", path, src, dest, ret));
exit(EXIT_FAILURE);
}
else {
bool progressing = false;
bool writeFailed = false;
EAscpState state = eStart;
const char y[] = "y\n";
const char n[] = "n\n";
int status = 0;
int w = 0;
int fd = pipefrom[0];
const char *answer = n;
String line;
StringInit(&line, NULL, 0, 0);
if (cache_key) {
answer = y;
}
{
int flags = fcntl(fd, F_GETFL, 0);
fcntl(fd, F_SETFL, flags | O_NONBLOCK);
}
close(pipeto[0]);
close(pipefrom[1]);
assert(sizeof y == sizeof n);
{
int hang = 0;
uint64_t prev = 0;
KTime_t tPrev = 0;
char sfSrc = 'B';
uint64_t hSrc = humanize(srcSz, &sfSrc, NULL);
int sig = 0;
uint64_t i = 0;
KDirectory *dir = NULL;
rc_t rc = KDirectoryNativeDir(&dir);
DISP_RC(rc, "KDirectoryNativeDir");
if (rc != 0) {
return rc;
}
while (w == 0) {
bool quit = false;
w = waitpid(nPid, &status, WNOHANG);
if (w == 0) {
bool got = false;
rc_t rc = 0;
if (quitting) {
rc = quitting();
}
if (rc != 0 || quit) {
if (sig == 0) {
sig = SIGINT;
}
else if (sig >= SIGKILL) {
break;
}
else {
++sig;
}
if (progressing) {
OUTMSG(("\n"));
}
PLOGMSG(klogInfo, (klogInfo, "^C pressed: "
"Senging $(sgn) to ascp", "sgn=%s", sig));
kill(nPid, sig);
}
while (true) {
char buf[4096];
int s = read(fd, buf, sizeof buf);
if (s == 0) {
break;
}
else if (s < 0) {
if (errno != EAGAIN) {
if (progressing) {
OUTMSG(("\n"));
}
perror("read(child)");
}
break;
}
ascpParse(buf, s, dest, &state, &line);
switch (state) {
case eKeyEnd:
write(pipeto[1], answer, sizeof y - 1);
break;
case eWriteFailed:
writeFailed = true;
break;
default:
break;
}
got = true;
}
if (!got) {
sleep(1);
++i;
if ((heartbeat > 0 && i >= heartbeat) || (i > 99)) {
uint64_t size = 0;
rc_t rc = KDirectoryFileSize(dir, &size, "%s", dest);
if (rc != 0) {
size = 0;
}
else {
if (size != prev) {
prev = size;
tPrev = 0;
hang = 0;
}
else {
KTime_t date = 0;
rc_t rc = KDirectoryDate(dir, &date, "%s", dest);
if (rc == 0) {
tPrev = date;
if ((KTimeStamp() - date) > 60 * 99) {
/* no file update during 99' */
if (hang == 0) {
write(pipeto[1],
answer, sizeof y - 1);
++hang;
}
else if (hang < 9) {
++hang;
sig = 0;
}
else {
if (sig == 0) {
sig = SIGINT;
}
else {
++sig;
}
if (progressing) {
OUTMSG(("\n"));
}
if (sig > SIGKILL) {
rc = RC(rcExe,
rcProcess, rcExecuting,
rcProcess,rcDetached);
return rc;
}
PLOGMSG(klogInfo, (klogInfo,
"Senging $(sgn) to ascp",
"sgn=%s", sig));
kill(nPid, sig);
}
}
}
}
}
if (heartbeat > 0) {
if (callback) {
quit = !callback(id,
eAscpStateRunning, size, 0);
}
else {
progress(acc, size, srcSz, hSrc, sfSrc,
tPrev);
}
progressing = true;
}
i = 0;
}
}
}
}
RELEASE(KDirectory, dir);
}
if (progressing) {
OUTMSG(("\n"));
}
while (1) {
char buf[4096];
int s = read(fd, buf, sizeof buf);
if (s == 0) {
break;
}
else if (s < 0) {
if (errno != EAGAIN) {
perror("read(child)");
break;
}
continue;
}
ascpParse(buf, s, dest, &state, &line);
if (state == eWriteFailed) {
writeFailed = true;
}
}
STSMSG( STS_DBG, ("ascp exited with pid=%d status=%d", w, status));
if (WIFEXITED(status)) {
STSMSG(STS_DBG, ("ascp exited with exit status %d",
WEXITSTATUS(status)));
}
else {
STSMSG(STS_DBG, ("ascp has not terminated correctly"));
}
if (w == -1) {
perror("waitpid");
rc = RC(rcExe, rcProcess, rcWaiting, rcProcess, rcFailed);
LOGERR(klogErr, rc, "after waitpid");
exit(EXIT_FAILURE);
}
if (WIFEXITED(status)) {
if (WEXITSTATUS(status) == 0) {
STSMSG(STS_DBG, ("ascp succeed"));
if (callback) {
callback(id, eAscpStateExitSuccess, 0, 0);
}
} else if (writeFailed) {
rc = RC(rcExe, rcProcess, rcExecuting, rcMemory, rcExhausted);
if (callback) {
callback(id, eAscpStateExitWriteFailure, 0, 0);
}
}
else {
if (rc == 0) {
rc = RC(rcExe, rcProcess, rcWaiting, rcProcess, rcFailed);
}
PLOGERR(klogErr, (klogErr, rc,
"ascp failed with $(ret)", "ret=%d", WEXITSTATUS(status)));
if (callback) {
callback(id, eAscpStateExitFailure, 0, 0);
}
}
} else if (WIFSIGNALED(status)) {
if (rc == 0) {
if (quitting) {
rc = quitting();
if (rc == 0) {
rc = RC(rcExe,
rcProcess, rcWaiting, rcProcess, rcFailed);
}
}
}
if (rc != SILENT_RC(rcExe, rcProcess, rcExecuting,
rcProcess, rcCanceled))
{
PLOGERR(klogErr, (klogErr, rc, "ascp killed by signal $(sig)",
"sig=%d", WTERMSIG(status)));
if (callback) {
callback(id, eAscpStateExitFailure, 0, 0);
}
}
}
}
return rc;
}
static
rc_t CC list_action (const KDirectory * dir, const char * path, void * _adata)
{
rc_t rc;
list_adata * data;
list_item * item;
KPathType type;
uint32_t access;
uint64_t size;
uint64_t loc;
KTime_t mtime;
size_t pathlen;
size_t linklen;
char link [2 * 4096]; /* we'll truncate? */
rc = 0;
data = _adata;
loc = size = 0;
pathlen = strlen (path);
type = KDirectoryPathType (dir, path);
if (type & kptAlias)
{
rc = KDirectoryVResolveAlias (dir, false, link, sizeof (link),
path, NULL);
if (rc == 0)
linklen = strlen (link);
}
else
{
linklen = 0;
switch (type & ~kptAlias)
{
case kptNotFound:
rc = RC (rcExe, rcDirectory, rcAccessing, rcPath, rcNotFound);
break;
case kptBadPath:
rc = RC (rcExe, rcDirectory, rcAccessing, rcPath, rcInvalid);
break;
case kptZombieFile:
if ( ! long_list )
return 0;
data->has_zombies = true;
case kptFile:
rc = KDirectoryFileSize (dir, &size, path);
if (rc == 0)
{
if (size > data->max_size)
data->max_size = size;
rc = KDirectoryFileLocator (dir, &loc, path);
if ((rc == 0) && (loc > data->max_loc))
data->max_loc = loc;
}
break;
case kptDir:
break;
case kptCharDev:
case kptBlockDev:
case kptFIFO:
/* shouldn't get here */
return 0;
}
}
if (rc == 0)
{
rc = KDirectoryAccess (dir, &access, "%s", path);
if (rc == 0)
{
rc = KDirectoryDate (dir, &mtime, "%s", path);
if (rc == 0)
{
item = malloc (sizeof (*item) + pathlen + linklen + 2); /* usually one too many */
if (item == NULL)
{
rc = RC (rcExe, rcNoTarg, rcAllocating, rcMemory, rcExhausted);
}
else
{
item->type = type;
item->access = access;
item->size = size;
item->loc = loc;
item->mtime = mtime;
item->path = (char *)(item+1);
strcpy (item->path, path);
if (type & kptAlias)
{
item->link = item->path + pathlen + 1;
strcpy (item->link, link);
}
else
item->link = NULL;
DLListPushHead (&data->list, &item->dad);
if (type == kptDir)
rc = step_through_dir (dir, path, data->filter, data->fdata,
list_action, data);
}
}
}
}
return rc;
}
LIB_EXPORT rc_t CC aspera_get(
const char *ascp_bin, const char *private_file, const char *src,
const char *dest, AscpOptions *opt)
{
AscpOptions dummy;
bool status = false;
uint64_t prev = 0;
int attempt = 0;
KDirectory *dir = NULL;
TQuitting *quitting = NULL;
rc_t rc = KDirectoryNativeDir(&dir);
if (rc != 0) {
return rc;
}
if (ascp_bin == NULL || private_file == NULL ||
src == NULL || dest == NULL)
{
return RC(rcNS, rcFile, rcCopying, rcParam, rcNull);
}
if (opt == NULL) {
memset(&dummy, 0, sizeof dummy);
opt = &dummy;
}
if (opt->target_rate == NULL) {
KConfig *cfg = NULL;
rc_t rc = KConfigMake(&cfg, NULL);
DISP_RC(rc, "cannot KConfigMake");
if (rc == 0) {
rc = _KConfigGetAscpRate(cfg, &opt->target_rate);
DISP_RC(rc, "cannot get aspera max rate");
}
RELEASE(KConfig, cfg);
}
sStatus = status = opt->status;
quitting = opt->quitting;
while (true) {
rc = run_ascp(ascp_bin, private_file, src, dest, opt);
if (rc == 0) {
if (status) {
STSMSG(STS_DBG, ("ascp finished with success"));
}
break;
}
else if (rc == SILENT_RC(rcExe,
rcProcess, rcExecuting, rcMemory, rcExhausted))
{
if (status) {
STSMSG(STS_DBG, ("ascp failed: %R", rc));
}
break;
}
else {
rc_t rc = 0;
uint64_t size = 0;
if (quitting != NULL) {
rc = quitting();
if (rc != 0) {
break;
}
}
if (status) {
STSMSG(STS_DBG, ("ascp failed: %R", rc));
}
rc = KDirectoryFileSize(dir, &size, dest);
if (rc != 0 || size < prev) {
if (status) {
STSMSG(0, ("KDirectoryFileSize after ascp run: "
"rc = %ld, size = %ld", rc, size));
}
break;
}
else if (size > prev) {
if (status) {
STSMSG(STS_INFO, (" fasp download failed. %ld bytes "
"received so far. Retrying...", size));
}
attempt = 0;
prev = size;
} else {
if (attempt++ > 3) {
break;
}
if (status) {
STSMSG(STS_INFO, (" fasp download failed. %ld bytes "
"received so far. Retrying %d...", size, attempt));
}
}
}
}
RELEASE(KDirectory, dir);
return rc;
}
static
rc_t list_action (const KDirectory * dir, const char * path, void * _adata)
{
rc_t rc = 0;
list_adata * data = _adata;
list_item * item = NULL;
KPathType type = KDirectoryPathType (dir, "%s", path);
size_t pathlen = strlen (path);
size_t linklen = 0;
char link [2 * 4096]; /* we'll truncate? */
if (type & kptAlias)
{
rc = KDirectoryVResolveAlias (dir, false, link, sizeof (link),
path, NULL);
if (rc == 0)
linklen = strlen (link);
}
if (rc == 0)
{
item = calloc (sizeof (*item) + pathlen + linklen + 2, 1); /* usually one too many */
if (item == NULL)
{
rc = RC (rcExe, rcNoTarg, rcAllocating, rcMemory, rcExhausted);
}
else
{
do
{
item->path = (char *)(item+1);
strcpy (item->path, path);
item->type = type;
rc = KDirectoryAccess (dir, &item->access, "%s", path);
if (rc) break;
rc = KDirectoryDate (dir, &item->mtime, "%s", path);
if (rc) break;
if (type & kptAlias)
{
item->link = item->path + pathlen + 1;
strcpy (item->link, link);
}
else switch (type & ~kptAlias)
{
case kptNotFound:
rc = RC (rcExe, rcDirectory, rcAccessing, rcPath, rcNotFound);
break;
case kptBadPath:
rc = RC (rcExe, rcDirectory, rcAccessing, rcPath, rcInvalid);
break;
case kptZombieFile:
data->has_zombies = true;
case kptFile:
rc = KDirectoryFileSize (dir, &item->size, "%s", path);
if (rc == 0)
rc = KDirectoryFileLocator (dir, &item->loc, "%s", path);
DBGMSG (DBG_APP, 1, ("%s: found file %s size %lu at %lu\n",
__func__, item->path, item->size, item->loc));
break;
case kptDir:
DBGMSG (DBG_APP, 1, ("%s: found directory %s\n",
__func__, item->path));
break;
default:
DBGMSG (DBG_APP, 1, ("%s: found unknown %s\n",
__func__, item->path));
break;
}
} while (0);
}
}
if (rc == 0)
{
VectorAppend (&data->list, NULL, item);
VectorInsert (&data->sort, item, NULL, list_item_cmp);
if (type == kptDir)
rc = step_through_dir (dir, path, list_action, data);
}
return rc;
}
static rc_t on_report_cache_file( visit_ctx * obj )
{
rc_t rc = 0;
uint64_t file_size = 0;
uint64_t used_size = 0;
uint64_t checked_blocks = 0;
uint64_t empty_blocks = 0;
float completeness = 0.0;
bool locked = false;
report_data * data = obj->data;
data->partial_count++;
rc = KDirectoryFileSize ( obj->dir, &file_size, "%s", obj->path );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
"KDirectoryFileSize( $(path) ) failed in $(func)", "path=%s,func=%s", obj->path, __func__ ) );
}
else
{
struct KFile const *f;
data->file_size += file_size;
rc = KDirectoryOpenFileRead ( obj->dir, &f, "%s", obj->path );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
"KDirectoryOpenFileRead( $(path) ) failed in $(func)", "path=%s,func=%s", obj->path, __func__ ) );
}
else
{
rc = GetCacheCompleteness( f, &completeness, &used_size );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
"GetCacheCompleteness( $(path) ) failed in $(func)", "path=%s,func=%s", obj->path, __func__ ) );
}
else
{
data->used_file_size += used_size;
}
if ( rc == 0 && obj->options->tstzero )
{
rc = Has_Cache_Zero_Blocks( f, &checked_blocks, &empty_blocks );
if ( rc != 0 )
{
PLOGERR( klogErr, ( klogErr, rc,
"Has_Cache_Zero_Blocks( $(path) ) failed in $(func)", "path=%s,func=%s", obj->path, __func__ ) );
}
}
KFileRelease( f );
}
}
if ( rc == 0 )
{
uint32_t pt = ( KDirectoryPathType ( obj->dir, "%s.lock", obj->path ) & ~ kptAlias );
locked = ( pt == kptFile );
}
if ( rc == 0 && obj->options->detailed )
{
if ( locked )
rc = KOutMsg( "%s complete by %.02f %% [%,u of %,u]bytes locked\n",
obj->path, completeness, used_size, file_size );
else
rc = KOutMsg( "%s complete by %.02f %% [%,u of %,u]bytes\n",
obj->path, completeness, used_size, file_size );
}
if ( rc == 0 && obj->options->tstzero )
{
rc = KOutMsg( "%s has %lu blocks set in bitmap where %lu are empty\n",
obj->path, checked_blocks, empty_blocks );
}
return rc;
}
