/* KMain - EXTERN
* executable entrypoint "main" is implemented by
* an OS-specific wrapper that takes care of establishing
* signal handlers, logging, etc.
*
* in turn, OS-specific "main" will invoke "KMain" as
* platform independent main entrypoint.
*
* "argc" [ IN ] - the number of textual parameters in "argv"
* should never be < 0, but has been left as a signed int
* for reasons of tradition.
*
* "argv" [ IN ] - array of NUL terminated strings expected
* to be in the shell-native character set: ASCII or UTF-8
* element 0 is expected to be executable identity or path.
*/
rc_t CC KMain ( int argc, char *argv [] )
{
Args * args;
rc_t rc;
KStsLevelSet (1);
rc = ArgsMakeAndHandle (&args, argc, argv, 1, Options,
sizeof (Options) / sizeof (Options[0]));
if (rc)
LOGERR (klogInt, rc, "failed to parse command line parameters");
else
{
uint32_t ocount;
rc = ArgsOptionCount (args, OPTION_DEC_SRA, &ocount);
if (rc)
LOGERR (klogInt, rc, "failed to examine decrypt "
"sra option");
else
{
DecryptSraFlag = (ocount > 0);
rc = CommonMain (args);
}
ArgsWhack (args);
}
if (rc)
STSMSG (1, ("exiting: %R (%u)", rc, rc));
else
STSMSG (1, ("exiting: success"));
return rc;
}
bool key_compare (const AES_KEY * control, const AES_KEY * test)
{
uint32_t ix;
uint32_t limit;
bool passed_rounds = true;
bool passed_keys = true;
passed_rounds = (control->rounds == test->rounds);
STSMSG (4,("rounds:\t%d : %d\t%s\n", control->rounds, test->rounds,
passed_string[passed_rounds]));
limit = (control->rounds + 1) * 4;
for (ix = 0; ix < limit; ++ix)
{
bool this_key;
this_key = ((control->rd_key[ix] == bswap_32(test->rd_key[ix])) ||
(control->rd_key[ix] == test->rd_key[ix]));
passed_keys = passed_keys && this_key;
STSMSG (4,("rd_key[%d]\t%0.8x : %0.8x\t%s\n",
ix, control->rd_key[ix], test->rd_key[ix],
passed_string[this_key]));
}
return passed_rounds && passed_keys;
}
bool DoThisFile (const KFile * infile, EncScheme enc, ArcScheme * parc)
{
const KFile * Infile;
ArcScheme arc;
rc_t rc;
*parc = arcNone;
switch (enc)
{
default:
STSMSG (1, ("not encrypted"));
return false;
case encEncFile:
/*
* this will apply to KEncFiles versions 1 and 2, maybe not 3
* but will hopefully become obsolete eventually.
*/
rc = KEncFileMakeRead (&Infile, infile, &Key); /* replace with VFSManagerOpenFileReadDirectoryRelativeInt */
if (rc)
return false;
break;
case encSraEncFile:
/* these are NCBIsenc instead of NCBInenc */
goto sra_enc_file;
case encWGAEncFile:
rc = KFileMakeWGAEncRead (&Infile, infile, Password, PasswordSize);
if (rc)
return false;
break;
}
arc = ArchiveTypeCheck (Infile);
KFileRelease (Infile);
switch (arc)
{
default:
return false;
case arcNone:
return true;
case arcSRAFile:
break;
}
sra_enc_file:
*parc = arcSRAFile;
STSMSG (1, ("encrypted sra archive\ndecryption%s requested",
DecryptSraFlag ? "" : " not"));
return DecryptSraFlag;
}
static
rc_t open_file_as_dir (const char * path, const KDirectory ** dir)
{
rc_t rc;
KPathType kpt;
const KDirectory * d;
rc = 0;
kpt = KDirectoryPathType (kdir, path);
switch (kpt & ~kptAlias)
{
case kptNotFound:
rc = RC (rcExe, rcNoTarg, rcParsing, rcParam, rcNotFound);
PLOGERR (klogFatal, (klogFatal, rc, "Archive [$(A)] must exist", PLOG_S(A), path));
return rc;
case kptDir:
default:
rc = RC (rcExe, rcNoTarg, rcParsing, rcParam, rcInvalid);
PLOGERR (klogFatal, (klogFatal, rc, "Archive [$(A)] must be a file", PLOG_S(A), path));
return rc;
case kptFile:
break;
}
if (rc == 0)
{
STSMSG (1, ("Opening %s\n", path));
rc = KDirectoryOpenArcDirRead (kdir, &d, /*chroot*/false, path,
tocKFile, KArcParseSRA,
/*filter*/pnamesFilter, /*filt.param*/NULL);
*dir = (rc == 0) ? d : NULL;
}
return rc;
}
bool CipherExample (KCipher * cipher)
{
uint8_t cipher_text [16];
rc_t rc;
bool passed_key;
bool passed_block;
memset (cipher_text, 0, sizeof cipher_text);
rc = KCipherSetEncryptKey (cipher, b.key, 128/8);
if (rc)
;
else
{
passed_key = key_compare (&b.key_schedule, cipher->encrypt_key);
rc = KCipherEncrypt (cipher, b.plain, cipher_text);
if (rc)
;
else
{
passed_block = block_compare (b.cipher, cipher_text);
}
}
if (rc)
{
STSMSG (1,("Could not run test"));
return false;
}
return passed_key && passed_block;
}
LIB_EXPORT rc_t CC ascp_locate(const char **ascp_bin, const char **private_file,
bool use_config, bool status)
{
static const char *bin = NULL;
static const char *key = NULL;
rc_t rc = 0;
KConfig *cfg = NULL;
bool found = false;
if (ascp_bin == NULL || private_file == NULL) {
return RC(rcNS, rcFile, rcCopying, rcParam, rcNull);
}
*ascp_bin = *private_file = NULL;
rc = returnAscp(bin, key, ascp_bin, private_file, &found);
if (rc != 0 || found) {
return rc;
}
rc = KConfigMake(&cfg, NULL);
if (rc != 0) {
return rc;
}
if (_KConfigAscpDisabled(cfg, status)) {
if (status) {
STSMSG(STS_INFO, ("Use of Aspera transfer is disabled "
"by the configuration, using HTTP transfer"));
}
}
else {
KDirectory *dir = NULL;
rc = _KConfigGetAscp(cfg, &bin, &key);
if (bin != NULL) {
assert(key && !rc);
RELEASE(KConfig, cfg);
rc = returnAscp(bin, key, ascp_bin, private_file, &found);
if (rc != 0 || found) {
return rc;
}
}
rc = KDirectoryNativeDir(&dir);
if (rc != 0) {
return rc;
}
while (ascp_path(&bin, &key)) {
if (_SystemHelp(bin, status)) {
if (_KDirectoryFileFound(dir, key, status)) {
break;
}
}
}
RELEASE(KDirectory, dir);
}
RELEASE(KConfig, cfg);
rc = returnAscp(bin, key, ascp_bin, private_file, &found);
if (rc != 0 || found) {
return rc;
}
else {
return RC(rcNS, rcFile, rcCopying, rcFile, rcNotFound);
}
}
static
rc_t copy_file (const KFile * fin, KFile *fout)
{
rc_t rc;
uint8_t buff [64 * 1024];
size_t num_read;
uint64_t inpos;
uint64_t outpos;
assert (fin != NULL);
assert (fout != NULL);
inpos = 0;
outpos = 0;
do
{
rc = KFileRead (fin, inpos, buff, sizeof (buff), &num_read);
if (rc != 0)
{
PLOGERR (klogErr, (klogErr, rc,
"Failed to read from directory structure in creating archive at $(P)",
PLOG_U64(P), inpos));
break;
}
else if (num_read > 0)
{
size_t to_write;
inpos += (uint64_t)num_read;
STSMSG (2, ("Read %zu bytes to %lu", num_read, inpos));
/* PLOGMSG (klogDebug10, "Read $(B) Bytes for $(T)", PLOG_2(PLOG_U64(B),PLOG_U64(T)), num_read, inpos); */
to_write = num_read;
while (to_write > 0)
{
size_t num_writ;
rc = KFileWrite (fout, outpos, buff, num_read, &num_writ);
if (rc != 0)
{
PLOGERR (klogErr, (klogErr, rc,
"Failed to write to archive in creating archive at $(P)",
PLOG_U64(P), outpos));
break;
}
outpos += num_writ;
/* PLOGMSG (klogDebug10, "Wrote $(B) Bytes for $(T)", PLOG_2(PLOG_U64(B),PLOG_U64(T)), num_writ, outpos); */
to_write -= num_writ;
}
}
/* else */
/* PLOGMSG (klogDebug10, "Read $(B) Bytes for $(T)", PLOG_2(PLOG_U64(B),PLOG_U64(T)), num_read, inpos); */
if (rc != 0)
break;
} while (num_read != 0);
return rc;
}
static bool _SystemHelp(const char *command, bool status) {
int value = 0;
if (status) {
STSMSG(STS_DBG, ("Checking '%s'", command));
}
value = _SilentSystem("\"%s\" -h", command);
if (value == 0) {
if (status) {
STSMSG(STS_INFO, ("Using '%s'", command));
}
return true;
}
else {
if (status) {
STSMSG(STS_DBG, ("'%s': not found", command));
}
return false;
}
}
/* KMain - EXTERN
* executable entrypoint "main" is implemented by
* an OS-specific wrapper that takes care of establishing
* signal handlers, logging, etc.
*
* in turn, OS-specific "main" will invoke "KMain" as
* platform independent main entrypoint.
*
* "argc" [ IN ] - the number of textual parameters in "argv"
* should never be < 0, but has been left as a signed int
* for reasons of tradition.
*
* "argv" [ IN ] - array of NUL terminated strings expected
* to be in the shell-native character set: ASCII or UTF-8
* element 0 is expected to be executable identity or path.
*/
rc_t CC KMain ( int argc, char *argv [] )
{
Args* args = NULL;
rc_t rc;
rc = ArgsMakeAndHandle(&args, argc, argv, 1, Options, sizeof Options / sizeof (OptDef));
if (rc)
LOGERR (klogInt, rc, "failed to parse command line parameters");
if (rc == 0)
{
uint32_t pcount;
bool force;
rc = ArgsOptionCount (args, OPTION_FORCE, &pcount);
if (rc)
LOGERR (klogInt, rc, "failed to examine force option");
else
{
force = (pcount > 0);
rc = ArgsParamCount (args, &pcount);
if (rc)
LOGERR (klogInt, rc, "failed to count parameters");
else
{
if (pcount != 2) {
MiniUsage (args);
rc = RC(rcApp, rcArgv, rcParsing, rcParam, rcInsufficient);
}
else
{
const char * src;
rc = ArgsParamValue (args, 0, (const void **)&src);
if (rc)
LOGERR (klogInt, rc, "failed to get source parameter");
else
{
const char * dst;
rc = ArgsParamValue (args, 1, (const void **)&dst);
if (rc)
LOGERR (klogInt, rc, "failed to get destination parameter");
else
{
rc = nenctool (src, dst, force);
}
}
}
}
}
ArgsWhack (args);
}
STSMSG (1, ("exiting: %R (%u)", rc, rc));
return rc;
}
LIB_EXPORT rc_t CC ascp_locate(const char **ascp_bin, const char **private_file,
bool use_config, bool status)
{
rc_t rc = 0;
KConfig *cfg = NULL;
if (ascp_bin == NULL || private_file == NULL) {
return RC(rcNS, rcFile, rcCopying, rcParam, rcNull);
}
*ascp_bin = *private_file = NULL;
rc = KConfigMake(&cfg, NULL);
if (rc != 0) {
return rc;
}
if (_KConfigAscpDisabled(cfg, status)) {
if (status) {
STSMSG(STS_INFO, ("Use of Aspera transfer is disabled "
"by the configuration, using HTTP transfer"));
}
}
else {
KDirectory *dir = NULL;
const char *bin = NULL;
const char *key = NULL;
rc = _KConfigGetAscp(cfg, ascp_bin, private_file);
if (*ascp_bin != NULL) {
assert(*private_file && !rc);
RELEASE(KConfig, cfg);
return 0;
}
rc = KDirectoryNativeDir(&dir);
if (rc != 0) {
return rc;
}
while (ascp_path(&bin, &key)) {
if (_SystemHelp(bin, status)) {
if (_KDirectoryFileFound(dir, key, status)) {
*ascp_bin = string_dup_measure(bin, NULL);
*private_file = string_dup_measure(key, NULL);
if (*ascp_bin == NULL || *private_file == NULL) {
free((void*)*ascp_bin);
free((void*)*private_file);
*ascp_bin = *private_file = NULL;
return RC(rcNS,
rcStorage, rcAllocating, rcMemory, rcExhausted);
}
break;
}
}
}
RELEASE(KDirectory, dir);
}
RELEASE(KConfig, cfg);
return rc;
}
static bool _KDirectoryFileFound(const KDirectory *self,
const char *path, bool status)
{
KPathType type = kptNotFound;
if (status) {
STSMSG(STS_DBG, ("Checking '%s'", path));
}
type = KDirectoryPathType(self, "%s", path);
if ((type & ~kptAlias) == kptFile) {
if (status) {
STSMSG(STS_DBG, ("'%s': found", path));
}
return true;
}
else {
if (status) {
STSMSG(STS_DBG, ("'%s': not found", path));
}
return false;
}
}
static bool _KConfigAscpDisabled(const KConfig *self, bool status) {
bool disabled = false;
const char path[] = "tools/ascp/disabled";
rc_t rc = KConfigReadBool(self, path, &disabled);
if (rc != 0) {
if (rc != SILENT_RC(rcKFG, rcNode, rcOpening, rcPath, rcNotFound)) {
DISP_RC(rc, path);
}
else {
if (status) {
STSMSG(STS_DBG, ("'%s': not found in configuration", path));
}
}
disabled = false;
}
else {
if (status) {
STSMSG(STS_DBG, ("'%s' = '%s'", path, disabled ? "true" : "false"));
}
}
return disabled;
}
rc_t CC KMain ( int argc, char *argv [] )
{
Args * args;
rc_t rc;
rc = ArgsMakeAndHandle (&args, argc, argv, 0);
if (rc == 0)
{
rc = run();
if (rc)
LOGERR (klogErr, rc, "Exiting failure");
else
STSMSG (0, ("Exiting okay\n"));
}
if (rc)
LOGERR (klogErr, rc, "Exiting status");
else
STSMSG (0, ("Existing status (%R)\n", rc));
return rc;
}
rc_t CC KMain ( int argc, char *argv [] )
{
Args * args;
rc_t rc;
rc = ArgsMakeAndHandle (&args, argc, argv, 0);
if (rc == 0)
{
STSMSG (3, ("Starting tests",__func__));
error_count = 0;
rc = run();
}
if (error_count)
STSMSG (0, ("Failed %d tests of %d run out of 28 possible",error_count, test_count));
else
STSMSG (0, ("Passed %d tests out of 28 possible\n", test_count));
if (rc)
LOGERR (klogErr, rc, "Exiting with a failure status");
exit (error_count);
}
static bool pnamesUsePath (const char * path)
{
bool ret;
STSMSG (4, ("Use path called %s", path));
STSMSG (4, ("Depth %u ", BSTreeDepth(&pnames, false) ));
if (BSTreeDepth(&pnames, false) == 0)
{
/* PLOGMSG (klogDebug9, "pnamesUsePath use $(P) by default", PLOG_S(P), path); */
ret = true;
}
else
{
ret = ( BSTreeFind (&pnames, path, pnameFindCmp ) != NULL);
/* if (ret) */
/* PLOGMSG (klogDebug9, "pnamesUsePath use $(P)", PLOG_S(P), path); */
/* else */
/* PLOGMSG (klogDebug9, "pnamesUsePath don't use $(P)", PLOG_S(P), path); */
}
STSMSG (4, ("Use? %u ", ret));
return ret;
}
bool KeyExpansion (KCipher * cipher, const key_expansion * ke)
{
rc_t rc;
rc = KCipherSetEncryptKey (cipher, ke->key, ke->Nk);
if (rc)
{
STSMSG (1,("Could not run test"));
return false;
}
else
{
return (key_compare (&ke->key_schedule, cipher->encrypt_key));
}
}
static
rc_t CommitIndex(KTable* ktbl, const char* name, const SLList* li)
{
SIndexData data;
STSMSG(0, ("Saving index %s", name));
data.rc = KTableCreateIndex(ktbl, &data.kidx, kitU64, kcmInit, name);
if( data.rc == 0 ) {
if( !SLListDoUntil(li, InsertIndexData, &data) ) {
data.rc = KIndexCommit(data.kidx);
}
KIndexRelease(data.kidx);
}
return data.rc;
}
static
rc_t run_kar_extract (const char * archive, const char * directory)
{
rc_t rc;
const KDirectory * din;
KDirectory * dout;
STSMSG (1, ("run_kar_extract"));
rc = open_file_as_dir (archive, &din);
if (rc == 0 )
{
char * directorystr;
rc = derive_directory_name (&directorystr, archive, directory);
if (rc != 0)
{
;
/* PLOGERR (klogDebug1, (klogDebug1, rc, "failure to derive archive [$(A)/$(D)]", */
/* PLOG_2(PLOG_S(A),PLOG_S(D)), archive, directory)); */
}
else
{
rc = open_out_dir (directorystr, &dout);
free (directorystr);
if (rc != 0)
{
LOGERR (klogErr, rc, "failure to open output directory");
}
else
{
extract_adata adata;
adata.dir = dout;
adata.filter = pnamesFilter;
adata.fdata = NULL;
rc = step_through_dir (din, ".", pnamesFilter, NULL, extract_action, &adata);
KDirectoryRelease (dout);
}
}
KDirectoryRelease (din);
}
return rc;
}
bool block_compare (const uint8_t control [16], const uint8_t test [16])
{
uint32_t ix;
bool passed = true;
for (ix = 0; ix < 16; ++ix)
{
bool passed_this;
passed_this = (control[ix] == test[ix]);
passed = passed && passed_this;
STSMSG (4,("byte %u: control %0.2x test %0.2x %s\n",
ix,
control[ix], test[ix],
passed_string[passed_this]));
}
return passed;
}
static
rc_t open_dir_as_archive (const char * path, const KFile ** file)
{
rc_t rc;
KPathType kpt;
const KDirectory * d;
assert (path != NULL);
assert (path[0] != '\0');
assert (file != NULL);
rc = 0;
kpt = KDirectoryPathType (kdir, path);
switch (kpt)
{
case kptFile:
STSMSG (1, ("Opening as archive %s\n", path));
rc = KDirectoryOpenSraArchiveRead (kdir, &d, false, path);
if (rc != 0)
rc = KDirectoryOpenTarArchiveRead (kdir, &d, false, path);
if (rc != 0)
rc = RC (rcExe, rcParam, rcOpening, rcDirectory, rcInvalid);
break;
default:
rc = RC (rcExe, rcNoTarg, rcParsing, rcParam, rcInvalid);
PLOGERR (klogFatal, (klogFatal, rc, "Parameter [$(P)] must be a directory", PLOG_S(P), path));
return rc;
case kptDir:
/* KOutMsg ("%s: opening dir\n",__func__); */
rc = KDirectoryVOpenDirRead (kdir, &d, false, path, NULL);
}
if (rc == 0)
{
/* KOutMsg ("%s: dir to archive\n",__func__); */
rc = KDirectoryOpenTocFileRead (d, file, alignment, pnamesFilter, NULL, sort_size_then_rel_path );
KDirectoryRelease (d);
}
return rc;
}
rc_t CC KMain ( int argc, char *argv [] )
{
Args * args;
rc_t rc;
rc = ArgsMakeAndHandle (&args, argc, argv, 0);
if (rc == 0)
{
rc = run();
ArgsRelease (args);
}
if (rc == 0 && num_errors != 0)
{
rc = RC(rcKrypto, rcBuffer, rcValidating, rcEncryption, rcIncorrect);
}
if (rc)
LOGERR (klogErr, rc, "Exiting status");
else
STSMSG (0, ("Exiting okay\n"));
return rc;
}
static
rc_t open_out_dir (const char * path, KDirectory ** dir)
{
rc_t rc;
KCreateMode mode;
rc = 0;
mode = force ? kcmParents|kcmInit : kcmParents|kcmCreate;
STSMSG (1, ("Creating %s\n", path));
rc = KDirectoryVCreateDir (kdir, 0777, mode, path, NULL);
if (rc != 0)
{
PLOGERR (klogFatal, (klogFatal, rc, "failure to create or clear output directory $(D)", PLOG_S(D), path));
}
else
{
rc = KDirectoryVOpenDirUpdate (kdir, dir, false, path, NULL);
if (rc != 0)
{
PLOGERR (klogFatal, (klogFatal, rc, "failure to open output directory $(D)", PLOG_S(D), path));
}
}
return rc;
}
static
rc_t run()
{
KDirectory * pwd;
rc_t rc;
STSMSG (1, ("Open file system\n"));
rc = KDirectoryNativeDir (&pwd);
if (rc)
LOGERR (klogErr, rc, "Failed to open filesystem");
else
{
const KFile * unencrypt;
STSMSG (1, ("Open unencryptd file %s\n", UNENCRYPT));
rc = KDirectoryOpenFileRead (pwd, &unencrypt, UNENCRYPT);
if (rc)
LOGERR (klogErr, rc, "failed to open unencryptd file");
else
{
const KFile * encrypt;
STSMSG (1, ("Open encryptd file %s\n", ENCRYPT));
rc = KDirectoryOpenFileRead (pwd, &encrypt, ENCRYPT);
if (rc)
LOGERR (klogErr, rc, "Failed to open encryptd file");
else
{
const KFile * decrypt;
STSMSG (1, ("Open decrypt file\n"));
rc = KFileMakeWGAEncRead (&decrypt, encrypt, WGA_KEY, 1);
if (rc)
LOGERR (klogErr, rc, "Failed to open decrypter");
else
{
size_t tot_readu;
size_t tot_readd;
size_t num_read;
uint8_t * decoded = NULL;
uint8_t * unencoded = NULL;
int comp;
decoded = calloc ( 64 * 1024, 24 );
unencoded = calloc ( 64 * 1024, 24 );
for (tot_readu = 0; rc == 0 && tot_readu < sizeof unencoded; tot_readu += num_read)
{
STSMSG (5, ("Read unencrypted '%u' @ %lu\n", sizeof unencoded - tot_readu, tot_readu));
rc = KFileRead (unencrypt, tot_readu, unencoded + tot_readu,
sizeof unencoded - tot_readu, &num_read);
if (num_read == 0)
break;
}
if (rc == 0)
{
for (tot_readd = 0; rc == 0 && tot_readd < sizeof decoded; tot_readd += num_read)
{
STSMSG (5, ("Read decrypted '%u' @ %lu\n", sizeof decoded - tot_readd, tot_readd));
rc = KFileRead (decrypt, tot_readd, decoded + tot_readd,
sizeof decoded - tot_readd, &num_read);
if (num_read == 0)
break;
}
comp = memcmp(decoded,unencoded, sizeof decoded);
STSMSG (1, ("Read u '%zu' d '%zu' cmp '%d'", tot_readu, tot_readd, comp));
if (comp != 0)
{
rc = RC (rcExe, rcNoTarg, rcValidating, rcFile, rcInconsistent);
LOGERR (klogErr, rc, "Unencryptfailed");
{
size_t ix;
size_t limit;
size_t matched = 0;
size_t mismatched = 0;
limit = tot_readu;
if (limit < tot_readd)
limit = tot_readd;
for (ix = 0; ix < limit; ++ix)
{
if (decoded[ix] != unencoded[ix])
{
++mismatched;
STSMSG (2, ("%5.5zu: D %c %2.2X U %c %2.2X\n",
ix,
decoded[ix]?decoded[ix]:'?',decoded[ix],
unencoded[ix]?unencoded[ix]:'?',unencoded[ix]));
}
else
++matched;
}
STSMSG (2, ("matched %zu mismatched %zu",matched,mismatched));
}
}
}
free (decoded);
free (unencoded);
}
KFileRelease (decrypt);
}
KFileRelease (encrypt);
}
KFileRelease (unencrypt);
}
KDirectoryRelease (pwd);
return rc;
}
LIB_EXPORT rc_t CC aspera_get(
const char *ascp_bin, const char *private_file, const char *aSrc,
const char *dest, AscpOptions *opt)
{
char path[PATH_MAX] = "";
AscpOptions dummy;
bool status = false;
int64_t prev = -1;
int attempt = 0;
KDirectory *dir = NULL;
TQuitting *quitting = NULL;
const char *src = aSrc;
rc_t rc = KDirectoryNativeDir(&dir);
if (rc != 0) {
return rc;
}
if (ascp_bin == NULL || private_file == NULL ||
src == NULL || dest == NULL)
{
return RC(rcNS, rcFile, rcCopying, rcParam, rcNull);
}
if (opt == NULL) {
memset(&dummy, 0, sizeof dummy);
opt = &dummy;
}
if (opt->ascp_options == NULL && opt->target_rate[0] == '\0') {
KConfig *cfg = NULL;
rc_t rc = KConfigMake(&cfg, NULL);
DISP_RC(rc, "cannot KConfigMake");
if (rc == 0) {
rc = _KConfigGetAscpRate(cfg,
opt->target_rate, sizeof opt->target_rate);
DISP_RC(rc, "cannot get aspera max rate");
}
RELEASE(KConfig, cfg);
}
sStatus = status = opt->status;
quitting = opt->quitting;
{
/* remove trailing #... or ?... from src path:
it could come from Revolver */
size_t s = string_size(aSrc);
const char *n = string_chr(aSrc, s, '#');
const char *q = string_chr(aSrc, s, '?');
if (q != NULL && (n == NULL || q < n)) {
n = q;
}
if (n != NULL) {
if (n - aSrc + 1 > sizeof path) {
return RC(rcNS, rcFile, rcCopying, rcBuffer, rcInsufficient);
}
else {
#if _DEBUGGING
size_t s =
#endif
string_copy(path, sizeof path, aSrc, n - aSrc);
assert(s <= sizeof path);
src = path;
}
}
}
while (true) {
rc = run_ascp(ascp_bin, private_file, src, dest, opt);
if (rc == 0) {
if (status) {
STSMSG(STS_DBG, ("ascp finished with success"));
}
break;
}
else if (rc == SILENT_RC(rcExe,
rcProcess, rcExecuting, rcMemory, rcExhausted))
{
if (status) {
STSMSG(STS_DBG, ("ascp failed: %R", rc));
}
break;
}
else {
rc_t rc2 = 0;
uint64_t size = 0;
if (quitting != NULL) {
rc2 = quitting();
if (rc2 != 0) {
break;
}
}
if (status) {
STSMSG(STS_DBG, ("ascp failed: %R", rc));
}
rc2 = KDirectoryFileSize(dir, &size, "%s", dest);
if (rc2 ==
SILENT_RC(rcFS, rcDirectory, rcAccessing, rcPath, rcNotFound))
{
if (prev < 0) {
if (status) {
STSMSG(0, ("fasp download failed. "
"File not found. Retrying..."));
}
prev = 0;
}
else {
if (status) {
STSMSG(0, ("fasp download failed. File not found."));
}
break;
}
}
else if (rc2 != 0 || (int64_t)size < prev) {
if (status) {
STSMSG(0, ("fasp download failed. KDirectoryFileSize "
"after ascp run: rc = %ld, size = %ld", rc, size));
}
break;
}
else if ((int64_t)size > prev) {
if (status) {
STSMSG(STS_INFO, (" fasp download failed. %ld bytes "
"received so far. Retrying...", size));
}
attempt = 0;
prev = size;
}
else {
if (attempt++ > 3) {
break;
}
if (status) {
STSMSG(STS_INFO, (" fasp download failed. %ld bytes "
"received so far. Retrying %d...", size, attempt));
}
}
}
}
RELEASE(KDirectory, dir);
return rc;
}
static
rc_t DoDir (const KDirectory * sd, KDirectory * dd)
{
KNamelist * names;
rc_t rc;
rc = KDirectoryList (sd, &names, NULL, NULL, ".");
if (rc)
;
else
{
uint32_t count;
rc = KNamelistCount (names, &count);
if (rc)
;
else
{
uint32_t idx;
for (idx = 0; idx < count; ++idx)
{
const char * name;
rc = KNamelistGet (names, idx, &name);
if (rc)
;
else
{
const KDirectory * nsd;
KDirectory * ndd;
KPathType kpt;
kpt = KDirectoryPathType (sd, name);
switch (kpt)
{
default:
break;
case kptFile:
if (sd == dd)
rc = FileInPlace (dd, name, true);
else
rc = FileToFile (sd, name, dd, name, true, NULL);
break;
case kptDir:
if (sd == dd)
{
rc = KDirectoryOpenDirUpdate (dd, &ndd, false, "%s", name);
if (rc)
;
else
{
/* RECURSION */
STSMSG (1, ("%scrypting directory %s", De, name));
rc = DoDir (ndd, ndd);
STSMSG (1, ("done with directory %s", name));
KDirectoryRelease (ndd);
}
}
else
{
rc = KDirectoryOpenDirRead (sd, &nsd, false, name);
if (rc)
;
else
{
rc = KDirectoryCreateDir (dd, 0600, kcmOpen, "%s", name);
if (rc)
;
else
{
rc = KDirectoryOpenDirUpdate (dd, &ndd, false, "%s", name);
if (rc)
;
else
{
/* RECURSION */
STSMSG (1, ("%scrypting directory %s", De, name));
rc = DoDir (nsd, ndd);
STSMSG (1, ("done with directory %s", name));
KDirectoryRelease (ndd);
}
}
KDirectoryRelease (nsd);
}
}
break;
}
}
}
}
KNamelistRelease (names);
}
return rc;
}
static
rc_t Start (KDirectory * cwd, const char * src, const char * dst)
{
KPathType dtype;
KPathType stype;
char dpath [MY_MAX_PATH];
char spath [MY_MAX_PATH];
rc_t rc;
bool using_stdin, using_stdout, try_rename;
/* limited anti oops checks */
try_rename = (dst == NULL);
if (!try_rename)
{
/* try to prevent file to file clash */
if (strcmp (src,dst) == 0)
dst = NULL;
/* try to prevent file to dir clash */
else
{
size_t s,d;
s = string_size (src);
d = string_size (dst);
if (s > d)
{
if (string_cmp (src, s, dst, d, d) == 0)
{
if ((strchr (src+d+1, '/') == NULL) &&
((src[d] == '/') ||
(src[d-1] == '/')))
{
try_rename = true;
dst = NULL;
}
}
}
}
}
/*
* This is a quick fix "hack"
* A fully built out VFS should replace the KFS in use and eliminate this
*/
using_stdin = (strcmp (src, "/dev/stdin") == 0);
if (using_stdin)
{
if (dst == NULL)
{
rc = RC (rcExe, rcArgv, rcParsing, rcParam, rcNull);
LOGERR (klogErr, rc, "Unable to handle stdin in place");
return rc;
}
stype = kptFile;
strcpy (spath, src);
UseStdin = true;
STSMSG (1, ("reading console / stdin as input"));
goto stdin_shortcut;
}
rc = KDirectoryResolvePath (cwd, false, spath, sizeof spath, "%s", src);
if (rc)
{
LOGERR (klogErr, rc, "can't resolve source");
return rc;
}
stype = KDirectoryPathType (cwd, spath);
switch (stype)
{
case kptNotFound:
rc = RC (rcExe, rcArgv, rcResolving, rcPath, rcNotFound);
break;
default:
case kptBadPath:
rc = RC (rcExe, rcArgv, rcResolving, rcPath, rcInvalid);
break;
case kptCharDev:
case kptBlockDev:
case kptFIFO:
case kptZombieFile:
case kptDataset:
case kptDatatype:
rc = RC (rcExe, rcArgv, rcResolving, rcPath, rcIncorrect);
break;
case kptFile:
case kptDir:
break;
}
if (rc)
{
PLOGERR (klogErr, (klogErr, rc, "can not use source '$(S)'", "S=%s", src));
return rc;
}
/*
* In Place Operation
*/
if (dst == NULL)
{
/*
* Input is a file
*/
if (stype == kptFile)
{
KDirectory * ndir;
char * pc;
pc = strrchr (spath, '/');
if (pc == NULL)
{
pc = spath;
ndir = cwd;
rc = KDirectoryAddRef (cwd);
}
else if (pc == spath)
{
++pc;
ndir = cwd;
rc = KDirectoryAddRef (cwd);
}
else
{
*pc++ = '\0';
rc = KDirectoryOpenDirUpdate (cwd, &ndir, false, spath);
}
if (rc == 0)
{
rc = FileInPlace (ndir, pc, try_rename);
KDirectoryRelease (ndir);
}
}
/*
* Input is a directory
*/
else
{
KDirectory * ndir;
rc = KDirectoryOpenDirUpdate (cwd, &ndir, false, spath);
if (rc)
;
else
{
STSMSG (1, ("%scrypting directory %s", De, spath));
rc = DoDir (ndir, ndir);
STSMSG (1, ("done with directory %s", spath));
KDirectoryRelease (ndir);
}
}
}
/*
* 'Copy' Operation
*/
else
{
stdin_shortcut:
using_stdout = (strcmp (dst, "/dev/stdout") == 0);
if (using_stdout == true)
{
dtype = kptFile;
strcpy (dpath, dst);
UseStdout = true;
STSMSG (1, ("writing console / stdout as output"));
goto do_file;
}
rc = KDirectoryResolvePath (cwd, false, dpath, sizeof dpath, "%s", dst);
if (rc)
{
LOGERR (klogErr, rc, "can't resolve destination");
return rc;
}
dtype = KDirectoryPathType (cwd, dpath);
switch (dtype)
{
default:
case kptBadPath:
rc = RC (rcExe, rcArgv, rcResolving, rcPath, rcInvalid);
PLOGERR (klogErr, (klogErr, rc, "can not use destination '$(S)'", "S=%s", dst));
break;
case kptCharDev:
case kptBlockDev:
case kptFIFO:
case kptZombieFile:
case kptDataset:
case kptDatatype:
rc = RC (rcExe, rcArgv, rcResolving, rcPath, rcIncorrect);
PLOGERR (klogErr, (klogErr, rc, "can not use destination parameter '$(S)'", "S=%s", dst));
break;
case kptNotFound:
{
size_t z;
z = strlen (dst) - 1;
if ((dst[z] == '/') || (stype == kptDir))
goto do_dir;
else
goto do_file;
}
case kptFile:
if (!ForceFlag)
{
rc = RC (rcExe, rcArgv, rcParsing, rcFile, rcExists);
PLOGERR (klogErr, (klogErr, rc, "can not over-write '$(F)' without --force",
"F=%s", dpath));
break;
}
do_file:
if (stype == kptFile)
{
rc = FileToFile (cwd, spath, cwd, dpath, try_rename, NULL);
}
else
{
rc = RC (rcExe, rcArgv, rcResolving, rcPath, rcIncorrect);
LOGERR (klogErr, rc, "Can't do directory to file");
}
break;
do_dir:
case kptDir:
/*
* Input is a directory
*/
if (stype == kptDir)
{
#if DIRECTORY_TO_DIRECTORY_SUPPORTED
const KDirectory * sdir;
KDirectory * ddir;
rc = KDirectoryOpenDirRead (cwd, &sdir, false, spath);
if (rc)
;
else
{
if (dtype == kptNotFound)
{
STSMSG (1, ("creating output directory %s", dpath));
rc = KDirectoryCreateDir (cwd, 0775, kcmCreate|kcmParents,
"%s", dpath);
}
if (rc == 0)
{
rc = KDirectoryOpenDirUpdate (cwd, &ddir, false, dpath);
if (rc)
;
else
{
STSMSG (1, ("%scrypting directory %s to %s", De, spath, dpath));
rc = DoDir (sdir, ddir);
STSMSG (1, ("done with directory %s to %s", spath, dpath));
KDirectoryRelease (ddir);
}
}
KDirectoryRelease (sdir);
}
#else
rc = RC (rcExe, rcArgv, rcResolving, rcPath, rcIncorrect);
LOGERR (klogErr, rc, "Can't do directory to directory");
#endif
}
/*
* Input is a file
*/
else
{
KDirectory * ndir;
const char * pc;
if (dtype == kptNotFound)
{
STSMSG (1, ("creating output directory %s", dpath));
rc = KDirectoryCreateDir (cwd, 0775, kcmCreate|kcmParents,
"%s", dpath);
}
if (rc == 0)
{
STSMSG (1, ("opening output directory %s", dpath));
rc = KDirectoryOpenDirUpdate (cwd, &ndir, false, dpath);
if (rc)
;
else
{
pc = strrchr (spath, '/');
if (pc == NULL)
pc = spath;
else
++pc;
rc = FileToFile (cwd, spath, ndir, pc, true, dpath);
KDirectoryRelease (ndir);
}
}
}
break;
}
}
return rc;
}
static
rc_t run (void)
{
VFSManager * manager;
rc_t rc = 0, orc = 0;
STSMSG (1, ("Make VFSManager"));
rc = VFSManagerMake (&manager);
STSMSG (2, ("rc %R", orc, orc));
if (rc == 0)
{
#if 1
static const char name[] = "test-kfs-manager-data-file";
#else
static const char name[] =
"ncbi-kfs:test-kfs-manager-data-file?enc&pwfile=password";
#endif
VPath * path;
STSMSG (1, ("Make test VPath file '%s'",name));
rc = VPathMake (&path, name);
STSMSG (2, ("rc %R", orc, orc));
if (rc == 0)
{
KFile * file;
STSMSG (1, ("Open File for write using manager and path"));
rc = VFSManagerCreateFile (manager, &file, false, 0666, kcmCreate,
path);
STSMSG (2, ("rc %R", rc, rc));
if (rc == 0)
{
char buff[4096];
size_t ix;
size_t num_writ;
uint64_t tot_writ = 0;
for (ix = 0; ix < sizeof buff; ++ix)
buff[ix] = 'A' + (ix%26);
STSMSG (1, ("writing to file"));
for (ix = 0; ix < 32; ++ix)
{
rc = KFileWrite (file, tot_writ, buff, sizeof buff, &num_writ);
if (rc == 0)
tot_writ += num_writ;
};
STSMSG (1, ("Release file - it should whack"));
orc = KFileRelease (file);
STSMSG (2, ("rc %R", orc, orc));
if (rc == 0) rc = orc;
#if 1
STSMSG (1, ("Remove file"));
orc = VFSManagerRemove (manager, true, path);
STSMSG (2, ("rc %R", orc, orc));
if (rc == 0) rc = orc;
#endif
}
STSMSG (1, ("Release VPath - it should Whack"));
orc = VPathRelease (path);
STSMSG (2, ("rc %R", orc, orc));
if (rc == 0) rc = orc;
}
STSMSG (1, ("Release VFSManager - it should Whack"));
orc = VFSManagerRelease (manager);
STSMSG (2, ("rc %R", orc, orc));
if (rc == 0) rc = orc;
}
return rc;
}
static rc_t Load( SParam* param )
{
rc_t rc = 0, rc1 = 0;
BSTree slides, evidence;
param->map_dir = NULL;
param->asm_dir = NULL;
param->output_dir = ( KDirectory * )param->input_dir;
BSTreeInit( &slides );
BSTreeInit( &evidence );
rc = open_dir_or_tar( param->input_dir, ¶m->map_dir, param->map_path );
if ( rc == 0 )
{
DirVisit_Data dv;
dv.param = param;
dv.tree = &slides;
dv.dir = param->map_dir;
rc = KDirectoryVisit( param->map_dir, true, DirVisitor, &dv, NULL );
if ( rc == 0 )
{
if ( param->asm_path != NULL )
{
rc_t rc2 = open_dir_or_tar( param->input_dir, ¶m->asm_dir, param->asm_path );
if ( rc2 == 0 )
{
dv.tree = &evidence;
dv.dir = param->asm_dir;
rc = KDirectoryVisit( param->asm_dir, true, DirVisitor, &dv, NULL );
}
}
if ( rc == 0 )
{
/* SHOULD HAVE A BSTreeEmpty FUNCTION OR SOMETHING...
MAKE ONE HERE - WITH KNOWLEDGE THAT TREE IS NOT NULL: */
#ifndef BSTreeEmpty
#define BSTreeEmpty( bst ) \
( ( bst ) -> root == NULL )
#endif
if ( BSTreeEmpty ( & slides ) && BSTreeEmpty ( & evidence ) )
rc = RC( rcExe, rcFile, rcReading, rcData, rcInsufficient );
else
{
/* CORRECTED SETTING OF "rc" IN "FGroupMAP_Validate" */
assert ( rc == 0 );
BSTreeForEach( &slides, false, FGroupMAP_Validate, &rc );
BSTreeForEach( &evidence, false, FGroupMAP_Validate, &rc );
}
if ( rc == 0 )
{
FGroupMAP_LoadData data;
PLOGMSG( klogInfo, ( klogInfo, "file set validation complete", "severity=status" ) );
memset( &data, 0, sizeof( data ) );
data.rc = 0;
data.param = param;
data.reads = &slides;
rc = DB_Init( param, &data.db );
if ( rc == 0 )
{
BSTreeDoUntil( &slides, false, FGroupMAP_LoadReads, &data );
rc = data.rc;
if ( rc == 0 )
{
PLOGMSG( klogInfo, ( klogInfo, "MAP loaded", "severity=status" ) );
BSTreeDoUntil( &evidence, false, FGroupMAP_LoadEvidence, &data );
rc = data.rc;
if ( rc == 0 )
PLOGMSG( klogInfo, ( klogInfo, "ASM loaded", "severity=status" ) );
}
}
rc1 = DB_Fini( param, &data.db, rc != 0 );
if ( rc == 0 )
rc = rc1;
}
}
}
/* copy the extra library ( file or recursive directory ) */
if ( rc == 0 && param->library != NULL )
{
const KDirectory *lib_src;
rc = open_dir_or_tar( param->input_dir, &lib_src, param->library );
if ( rc == 0 )
{
rc = copy_library( param->input_dir, param->output_dir,
param->library, param->out );
if ( rc == 0 )
STSMSG( 0, ( "extra lib copied" ) );
else
LOGERR( klogErr, rc, "failed to copy extra library" );
KDirectoryRelease( lib_src );
}
/*
else
{
rc = copy_library( param->input_dir, param->output_dir,
".", param->out );
if ( rc == 0 )
STSMSG( 0, ( "extra lib copied" ) );
else
LOGERR( klogErr, rc, "failed to copy extra library" );
}
*/
}
KDirectoryRelease( param->map_dir );
KDirectoryRelease( param->asm_dir );
}
BSTreeWhack( &slides, FGroupMAP_Whack, NULL );
BSTreeWhack( &evidence, FGroupMAP_Whack, NULL );
return rc;
}
/* KMain
*/
rc_t CC KMain ( int argc, char *argv [] )
{
Args * args;
rc_t rc;
rc = ArgsMakeAndHandle (&args, argc, argv, 0);
if (rc)
LOGERR (klogInt, rc, "failed to parse arguments");
else do
{
uint32_t acount;
rc = ArgsParamCount (args, &acount);
if (rc)
{
LOGERR (klogInt, rc, "failed to count parameters");
break;
}
if (acount == 0)
{
rc = MiniUsage (args);
break;
}
else
{
VFSManager* mgr;
rc = VFSManagerMake(&mgr);
if (rc)
LOGERR ( klogErr, rc, "failed to create VFSManager object" );
else
{
VResolver * resolver;
rc = VFSManagerGetResolver (mgr, &resolver);
if (rc == 0)
{
uint32_t ix;
for ( ix = 0; ix < acount; ++ ix )
{
const char * pc;
rc = ArgsParamValue (args, ix, &pc );
if (rc)
LOGERR (klogInt, rc,
"failed to retrieve parameter value");
else
{
const VPath * upath = NULL;
rc = VFSManagerMakePath ( mgr, (VPath**)&upath, "%s", pc);
if (rc == 0)
{
const VPath * local;
const VPath * remote;
rc = VResolverQuery (resolver, eProtocolHttp, upath, &local, &remote, NULL);
if (rc == 0)
{
const String * s;
if (local != NULL)
rc = VPathMakeString (local, &s);
else
rc = VPathMakeString (remote, &s);
if (rc == 0)
{
OUTMSG (("%S\n", s));
free ((void*)s);
}
VPathRelease (local);
VPathRelease (remote);
}
else
{
KDirectory * cwd;
rc_t orc = VFSManagerGetCWD (mgr, &cwd);
if (orc == 0)
{
KPathType kpt
= KDirectoryPathType(cwd, "%s", pc);
switch (kpt &= ~kptAlias)
{
case kptNotFound:
STSMSG(1, ("'%s': not found while "
"searching the file system",
pc));
break;
case kptBadPath:
STSMSG(1, ("'%s': bad path while "
"searching the file system",
pc));
break;
default:
STSMSG(1, ("'%s': "
"found in the file system",
pc));
rc = 0;
break;
}
}
if (orc == 0 && rc == 0) {
if (rc != 0) {
PLOGMSG(klogErr, (klogErr,
"'$(name)': not found",
"name=%s", pc));
}
else {
char resolved[PATH_MAX] = "";
rc = KDirectoryResolvePath(cwd, true,
resolved, sizeof resolved, "%s", pc);
if (rc == 0) {
STSMSG(1, ("'%s': found in "
"the current directory at '%s'",
pc, resolved));
OUTMSG (("%s\n", resolved));
}
else {
STSMSG(1, ("'%s': cannot resolve "
"in the current directory",
pc));
OUTMSG (("./%s\n", pc));
}
}
}
KDirectoryRelease(cwd);
}
}
RELEASE(VPath, upath);
}
}
VResolverRelease (resolver);
}
VFSManagerRelease(mgr);
}
}
ArgsWhack (args);
} while (0);
return rc;
}
rc_t ascpParse(const char *buf, size_t len, const char *filename,
EAscpState *state, String *line)
{
bool failure = false;
const char *p = buf;
int64_t l = len;
assert(buf && len && filename && state && line);
StringInit(line, NULL, 0, 0);
while (true) {
const char *n = string_chr(p, l, '\n');
const char *r = string_chr(p, l, '\r');
if (n == NULL) {
if (r != NULL) {
n = r;
}
}
else {
if (r != NULL) {
if (r < n) {
n = r;
}
}
}
if (n != NULL) {
StringInit(line, p, n - p, (uint32_t)(n - p));
l -= n - p + 1;
}
else {
StringInit(line, p, l, (uint32_t)l);
}
if (line->addr && line->len && line->addr[line->len - 1] == '\r') {
line->size = line->len - 1;
line->len = line->len - 1;
}
if (line->addr && line->len && line->addr[0] == '\r') {
++line->addr;
line->size = line->len - 1;
line->len = line->len - 1;
}
if (line->len != 0) {
SAscpState full;
rc_t rc = parseAscpLine(line, &full, filename);
if (rc != 0) {
return rc;
}
switch (full.s) {
case eChild:
break;
case eUnknown:
switch (*state) {
case eKeyStart:
case eKeyMayBeIn:
case eKeyIn:
*state = eKeyMayBeIn;
break;
case eCompleted:
case eFailed:
case eWriteFailed:
*state = eEnd;
/* report to user */
break;
case eProgress:
if (sStatus) {
OUTMSG(("\n"));
}
/* no break; */
default:
*state = eUnknown;
/* report to user */
break;
}
break;
case eFailed:
if (*state == eProgress) {
if (sStatus) {
OUTMSG(("\n"));
}
}
failure = true;
*state = full.s;
if (sStatus) {
OUTMSG(("%s\n", full.msg));
}
/* no break; */
break;
case eWriteFailed:
if (*state == eProgress) {
if (sStatus) {
OUTMSG(("\n"));
}
}
failure = true;
*state = full.s;
if (sStatus) {
OUTMSG(("%s\n", full.msg));
}
/* no break; */
break;
case eCompleted:
if (*state == eProgress) {
if (sStatus) {
OUTMSG(("\n"));
}
}
failure = false;
*state = full.s;
if (sStatus) {
OUTMSG(("%s\n", full.msg));
}
/* no break; */
break;
case eProgress:
if (*state == eProgress) {
if (sStatus) {
OUTMSG(("\r"));
}
}
*state = full.s;
if (sStatus) {
OUTMSG(("%s", full.msg));
}
break;
case eEnd:
if (*state == eProgress) {
if (sStatus) {
OUTMSG(("\n"));
}
}
*state = full.s;
if (sStatus) {
OUTMSG(("%s\n", full.msg));
}
/* report to user */
break;
default:
*state = full.s;
break;
}
SAscpStateFini(&full);
}
if (n == NULL || l <= 0) {
break;
}
if (*state == eKeyEnd) {
String end;
if (_StringHas(line, "Store key in cache? (y/n) ", &end)) {
if (n > end.addr + end.len) {
l += n - end.addr + end.len;
n = end.addr + end.len - 1;
}
}
}
p = n + 1;
if (p >= buf + len) {
break;
}
}
if (sStatus) {
STSMSG(STS_FIN, ("%.*s", len, buf));
}
if (failure) {
/* ignore it */
}
return 0;
}
