static int read_metadata_block(const struct PkgData *pdata,
const long long start, long long *next,
void *buf, const size_t buf_size)
{
long long offset = start;
unsigned short c_byte;
int ret = read_fs_bytes(pdata->fd, offset, &c_byte, 2);
if (ret) {
goto failed;
}
offset += 2;
int csize = SQUASHFS_COMPRESSED_SIZE(c_byte);
TRACE("read_metadata_block: block @0x%llx, %d %s bytes\n", start, csize,
SQUASHFS_COMPRESSED(c_byte) ? "compressed" : "uncompressed");
ret = SQUASHFS_COMPRESSED(c_byte)
? read_compressed(pdata, offset, csize, buf, buf_size)
: read_uncompressed(pdata, offset, csize, buf, buf_size);
if (ret < 0) {
goto failed;
}
offset += csize;
if (next) *next = offset;
return ret;
failed:
ERROR("Failed to read metadata block @0x%llx\n", start);
return ret;
}
static int read_data_block(const struct PkgData *pdata,
void *buf, const size_t buf_size,
const long long offset, const unsigned int c_byte)
{
const size_t csize = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte);
return SQUASHFS_COMPRESSED_BLOCK(c_byte)
? read_compressed(pdata, offset, csize, buf, buf_size)
: read_uncompressed(pdata, offset, csize, buf, buf_size);
}
bool detect_format_and_read(config &cfg, std::istream &in, std::string* error_log)
{
unsigned char c = in.peek();
if (c < 4) {
read_compressed(cfg, in);
return true;
} else {
read(cfg, in, error_log);
return false;
}
}
// read “size” bytes from the file after moving
// “offset” through the file, use math to determine the block
static int tar_read(const char *path, char *buf, size_t size, off_t offset,
struct fuse_file_info *fi)
{
int errornumber = 0;
// connect to database, begin a transaction
MYSQL *con = mysql_init(NULL);
//read options from file
mysql_options(con, MYSQL_READ_DEFAULT_FILE, SQLCONFILE); //SQLCONFILE defined in sqloptions.h
mysql_options(con, MYSQL_READ_DEFAULT_GROUP, SQLGROUP);
if(!mysql_real_connect(con, NULL, NULL, NULL, NULL, 0, NULL, 0)) {
//exit, connection failed
mysql_close(con);
errornumber = -EIO;
return errornumber;
}
char* archivename = NULL;
char* within_tar_path = NULL;
char* within_tar_filename = NULL;
char* file_ext = NULL;
parsepath(path, &archivename, &within_tar_path, &within_tar_filename, &file_ext);
char insQuery[1000];
// read specific variables
char path_to_archive[5000]; //stored path to the real location of the archive
struct blockmap* block_offsets = (struct blockmap*) malloc(sizeof(struct blockmap));
block_offsets->blocklocations = NULL; // map of blocks in archive
off_t real_offset; // offset of file within block + offset from beginning of file
size_t real_size; // the real amount to read min("size" , "filesize - offset")
// path is "/"
if(archivename == NULL) {
errornumber = -EACCES;
}
// path is "/TarArchive.tar" or "/TarArchive.tar.bz2" or "/TarArchive.tar.xz"
else if(within_tar_path == NULL) {
long long int archiveid = 0 + fi->fh; //get archive id from file_info
sprintf(insQuery, "SELECT ArchivePath, Timestamp from ArchiveList WHERE ArchiveID = %lld", archiveid);
if(mysql_query(con, insQuery)) {
//query error
errornumber = -EIO;
}
else {
MYSQL_RES* result = mysql_store_result(con);
if(result == NULL) {
errornumber = -EIO;
}
else {
if(mysql_num_rows(result) == 0) {
//file does not exist, set not found error
errornumber = -ENOENT;
}
else {
MYSQL_ROW row = mysql_fetch_row(result);
struct stat statbuff;
if(lstat(row[0], &statbuff) == -1) {
errornumber = -errno;
}
//check timestamp
else {
char* mod_time = ctime(&(statbuff.st_mtime));
if(strcmp(row[1], mod_time) != 0) {
errornumber = -ENOENT;
}
// open and read file
else {
int fi_des = open(row[0], O_RDONLY);
if (fi_des == -1)
errornumber = -errno;
else {
if (pread(fi_des, buf, size, offset) == -1) errornumber = -errno;
close(fi_des);
}
}
}
}
mysql_free_result(result);
}
}
}
// path is /TarArchive.tar/more
else {
/* Get Archive's real path ********************************************/
sprintf(insQuery, "SELECT ArchivePath, Timestamp from ArchiveList WHERE ArchiveName = '%s'", archivename);
if(mysql_query(con, insQuery)) {
//query error
errornumber = -EIO;
}
else {
MYSQL_RES* result = mysql_store_result(con);
if(result == NULL) {
errornumber = -EIO;
}
else {
if(mysql_num_rows(result) == 0) {
//file does not exist, set not found error
errornumber = -ENOENT;
}
else {
MYSQL_ROW row = mysql_fetch_row(result);
struct stat statbuff;
if(lstat(row[0], &statbuff) == -1) {
errornumber = -errno;
}
//check timestamp
else {
char* mod_time = ctime(&(statbuff.st_mtime));
if(strcmp(row[1], mod_time) != 0) {
errornumber = -ENOENT;
}
// copy filepath to easy location
else {
strcpy(path_to_archive, row[0]);
}
}
}
mysql_free_result(result);
}
}
/* Load Blockmap ******************************************************/
if(errornumber == 0) {
int needBlockmap = 0;
//no file extension
if(file_ext == NULL) errornumber = -ENOENT;
//.tar
else if(strcmp(".tar", file_ext) == 0) {
needBlockmap = 0;
}
//.bz2 //TODO add other forms of bz2 extention
else if(strcmp(".bz2", file_ext) == 0) {
needBlockmap = 1;
sprintf(insQuery, "SELECT Blocknumber, BlockOffset, BlockSize from Bzip2_blocks WHERE ArchiveName = '%s'", archivename);
}
//.xz or .txz
else if(strcmp(".xz", file_ext) == 0 || strcmp(".txz", file_ext) == 0) {
needBlockmap = 1;
sprintf(insQuery, "SELECT Blocknumber, BlockOffset, BlockSize from CompXZ_blocks WHERE ArchiveName = '%s'", archivename);
}
//unrecognized file extension
else errornumber = -ENOENT;
//if no error and we need a blockmap send query and use result
if(errornumber == 0 && needBlockmap == 1) {
if(mysql_query(con, insQuery)) {
//query error
errornumber = -EIO;
}
else {
MYSQL_RES* result = mysql_store_result(con);
if(result == NULL) {
errornumber = -EIO;
}
else {
int number_of_results = mysql_num_rows(result);
if(number_of_results == 0) {
//no results
errornumber = -ENOENT;
}
else {
MYSQL_ROW row;
//while there are rows to be fetched
block_offsets->blocklocations = (struct blocklocation*) malloc(sizeof(struct blocklocation) * (number_of_results + 10)); //slightly too large to be safe
block_offsets->maxsize = (number_of_results + 10);
while((row = mysql_fetch_row(result))) {
long int this_block_num = strtol(row[0], NULL, 10);
unsigned long long this_pos = strtoull(row[1], NULL, 10);
unsigned long long this_unC_size = strtoull(row[2], NULL, 10);
((block_offsets->blocklocations)[this_block_num]).position = this_pos;
((block_offsets->blocklocations)[this_block_num]).uncompressedSize = this_unC_size;
}
}
mysql_free_result(result);
}
}
}
}
/************** get info about file you want to extract and do the read *******/
if(errornumber == 0) {
long long int files_id = 0 + fi->fh; //get archive id from file_info
//no file extension
if(file_ext == NULL) errornumber = -ENOENT;
//.tar
else if(strcmp(".tar", file_ext) == 0) {
sprintf(insQuery, "SELECT GBoffset, BYTEoffset, MemberLength from UncompTar WHERE FileID = %lld", files_id);
if(mysql_query(con, insQuery)) {
//query error
errornumber = -EIO;
}
else {
MYSQL_RES* result = mysql_store_result(con);
if(result == NULL) {
errornumber = -EIO;
}
else {
if(mysql_num_rows(result) == 0) {
//no results
errornumber = -ENOENT;
}
else {
MYSQL_ROW row;
row = mysql_fetch_row(result);
unsigned long long length_of_file = strtoull(row[2], NULL, 10);
//check if offset puts us outside file range
if(offset >= length_of_file) errornumber = -ENXIO;
else {
//calculate real offset in tarfile
real_offset = (strtoull(row[0], NULL, 10) * BYTES_IN_GB) + strtoull(row[1], NULL, 10) + offset;
//calculate real size to be read
real_size = length_of_file - offset;
if(size < real_size) {
real_size = size;
}
int fi_des = open(path_to_archive, O_RDONLY);
if (fi_des == -1)
errornumber = -errno;
else {
unsigned long long Re = pread(fi_des, buf, real_size, real_offset);
if (Re == -1) errornumber = -errno;
else errornumber = Re;
close(fi_des);
}
}
}
mysql_free_result(result);
}
}
}
//TODO.bz2 //TODO add other forms of bz2 extention
else if(strcmp(".bz2", file_ext) == 0 || strcmp(".xz", file_ext) == 0 || strcmp(".txz", file_ext) == 0) {
int fileflag;
if(strcmp(".bz2", file_ext) == 0) {
sprintf(insQuery, "SELECT Blocknumber, InsideOffset, MemberLength from Bzip2_files WHERE FileID = %lld", files_id);
fileflag = BZ2FLAG;
}
else {
sprintf(insQuery, "SELECT Blocknumber, InsideOffset, MemberLength from CompXZ WHERE FileID = %lld", files_id);
fileflag = XZFLAG;
}
if(mysql_query(con, insQuery)) {
//query error
errornumber = -EIO;
}
else {
MYSQL_RES* result = mysql_store_result(con);
if(result == NULL) {
errornumber = -EIO;
}
else {
if(mysql_num_rows(result) == 0) {
//no results
errornumber = -ENOENT;
}
else {
MYSQL_ROW row;
row = mysql_fetch_row(result);
unsigned long long length_of_file = strtoull(row[2], NULL, 10);
//check if offset puts us outside file range
if(offset >= length_of_file) errornumber = -ENXIO;
else {
//find real block and real offset
long long blocknumber = strtoll(row[0], NULL, 10);
unsigned long long real_offset = 0 + strtoull(row[1], NULL, 10);
unsigned long long data_remaining = ((block_offsets->blocklocations)[blocknumber]).uncompressedSize - real_offset;
//calculate real size to be read
real_size = length_of_file - offset;
if(size < real_size) {
real_size = size;
}
//move real_offset foward by "offset"
unsigned long long remaining_seek = 0 + offset;
printf("remaining_seek: %llu\n", remaining_seek);
printf("blocknumber: %lld\n", blocknumber);
while(remaining_seek > data_remaining) {
blocknumber++;
remaining_seek = remaining_seek - data_remaining;
data_remaining = ((block_offsets->blocklocations)[blocknumber]).uncompressedSize;
real_offset = 0;
}
if(remaining_seek != 0) {
real_offset = 0 + remaining_seek;
}
//blocknumber and real_offset now point to the offset within the file desired by read
unsigned long long bn = 0+blocknumber;
unsigned long long ro = 0+real_offset;
unsigned long long rs = 0+real_size;
printf("block %llu, offset %llu, size %llu\n", bn, ro, rs);
errornumber = read_compressed(path_to_archive, fileflag, blocknumber, real_offset, real_size, buf, block_offsets);
}
}
mysql_free_result(result);
}
}
}
//unrecognized file extension
else errornumber = -ENOENT;
}
}
//free possible mallocs and mysql connection
mysql_close(con);
if(archivename != NULL) free(archivename);
if(within_tar_path != NULL) free(within_tar_path);
if(within_tar_filename != NULL) free(within_tar_filename);
if(block_offsets->blocklocations != NULL) free(block_offsets->blocklocations);
if(block_offsets != NULL) free(block_offsets);
return errornumber;
}
/**
* cdk_pkt_read:
* @inp: the input stream
* @pkt: allocated packet handle to store the packet
*
* Parse the next packet on the @inp stream and return its contents in @pkt.
**/
cdk_error_t
cdk_pkt_read (cdk_stream_t inp, cdk_packet_t pkt)
{
int ctb, is_newctb;
int pkttype;
size_t pktlen = 0, pktsize = 0, is_partial = 0;
cdk_error_t rc;
if (!inp || !pkt)
return CDK_Inv_Value;
ctb = cdk_stream_getc (inp);
if (cdk_stream_eof (inp) || ctb == EOF)
return CDK_EOF;
else if (!ctb)
return CDK_Inv_Packet;
pktsize++;
if (!(ctb & 0x80))
{
_cdk_log_info ("cdk_pkt_read: no openpgp data found. "
"(ctb=%02X; fpos=%02X)\n", ctb, cdk_stream_tell (inp));
return CDK_Inv_Packet;
}
if (ctb & 0x40) /* RFC2440 packet format. */
{
pkttype = ctb & 0x3f;
is_newctb = 1;
}
else /* the old RFC1991 packet format. */
{
pkttype = ctb & 0x3f;
pkttype >>= 2;
is_newctb = 0;
}
if (pkttype > 63)
{
_cdk_log_info ("cdk_pkt_read: unknown type %d\n", pkttype);
return CDK_Inv_Packet;
}
if (is_newctb)
read_new_length (inp, &pktlen, &pktsize, &is_partial);
else
read_old_length (inp, ctb, &pktlen, &pktsize);
pkt->pkttype = pkttype;
pkt->pktlen = pktlen;
pkt->pktsize = pktsize + pktlen;
pkt->old_ctb = is_newctb ? 0 : 1;
rc = 0;
switch (pkt->pkttype)
{
case CDK_PKT_ATTRIBUTE:
pkt->pkt.user_id = cdk_calloc (1, sizeof *pkt->pkt.user_id
+ pkt->pktlen + 16 + 1);
if (!pkt->pkt.user_id)
return CDK_Out_Of_Core;
pkt->pkt.user_id->name = (char*)pkt->pkt.user_id + sizeof(*pkt->pkt.user_id);
rc = read_attribute (inp, pktlen, pkt->pkt.user_id);
pkt->pkttype = CDK_PKT_ATTRIBUTE;
break;
case CDK_PKT_USER_ID:
pkt->pkt.user_id = cdk_calloc (1, sizeof *pkt->pkt.user_id
+ pkt->pktlen + 1);
if (!pkt->pkt.user_id)
return CDK_Out_Of_Core;
pkt->pkt.user_id->name = (char*)pkt->pkt.user_id + sizeof(*pkt->pkt.user_id);
rc = read_user_id (inp, pktlen, pkt->pkt.user_id);
break;
case CDK_PKT_PUBLIC_KEY:
pkt->pkt.public_key = cdk_calloc (1, sizeof *pkt->pkt.public_key);
if (!pkt->pkt.public_key)
return CDK_Out_Of_Core;
rc = read_public_key (inp, pktlen, pkt->pkt.public_key);
break;
case CDK_PKT_PUBLIC_SUBKEY:
pkt->pkt.public_key = cdk_calloc (1, sizeof *pkt->pkt.public_key);
if (!pkt->pkt.public_key)
return CDK_Out_Of_Core;
rc = read_public_subkey (inp, pktlen, pkt->pkt.public_key);
break;
case CDK_PKT_SECRET_KEY:
pkt->pkt.secret_key = cdk_calloc (1, sizeof *pkt->pkt.secret_key);
if (!pkt->pkt.secret_key)
return CDK_Out_Of_Core;
pkt->pkt.secret_key->pk = cdk_calloc (1,
sizeof *pkt->pkt.secret_key->pk);
if (!pkt->pkt.secret_key->pk)
return CDK_Out_Of_Core;
rc = read_secret_key (inp, pktlen, pkt->pkt.secret_key);
break;
case CDK_PKT_SECRET_SUBKEY:
pkt->pkt.secret_key = cdk_calloc (1, sizeof *pkt->pkt.secret_key);
if (!pkt->pkt.secret_key)
return CDK_Out_Of_Core;
pkt->pkt.secret_key->pk = cdk_calloc (1,
sizeof *pkt->pkt.secret_key->pk);
if (!pkt->pkt.secret_key->pk)
return CDK_Out_Of_Core;
rc = read_secret_subkey (inp, pktlen, pkt->pkt.secret_key);
break;
case CDK_PKT_LITERAL:
pkt->pkt.literal = cdk_calloc (1, sizeof *pkt->pkt.literal);
if (!pkt->pkt.literal)
return CDK_Out_Of_Core;
rc = read_literal (inp, pktlen, &pkt->pkt.literal, is_partial);
break;
case CDK_PKT_ONEPASS_SIG:
pkt->pkt.onepass_sig = cdk_calloc (1, sizeof *pkt->pkt.onepass_sig);
if (!pkt->pkt.onepass_sig)
return CDK_Out_Of_Core;
rc = read_onepass_sig (inp, pktlen, pkt->pkt.onepass_sig);
break;
case CDK_PKT_SIGNATURE:
pkt->pkt.signature = cdk_calloc (1, sizeof *pkt->pkt.signature);
if (!pkt->pkt.signature)
return CDK_Out_Of_Core;
rc = read_signature (inp, pktlen, pkt->pkt.signature);
break;
case CDK_PKT_PUBKEY_ENC:
pkt->pkt.pubkey_enc = cdk_calloc (1, sizeof *pkt->pkt.pubkey_enc);
if (!pkt->pkt.pubkey_enc)
return CDK_Out_Of_Core;
rc = read_pubkey_enc (inp, pktlen, pkt->pkt.pubkey_enc);
break;
case CDK_PKT_COMPRESSED:
pkt->pkt.compressed = cdk_calloc (1, sizeof *pkt->pkt.compressed);
if (!pkt->pkt.compressed)
return CDK_Out_Of_Core;
rc = read_compressed (inp, pktlen, pkt->pkt.compressed);
break;
case CDK_PKT_MDC:
pkt->pkt.mdc = cdk_calloc (1, sizeof *pkt->pkt.mdc);
if (!pkt->pkt.mdc)
return CDK_Out_Of_Core;
rc = read_mdc (inp, pkt->pkt.mdc);
break;
default:
/* Skip all packets we don't understand */
skip_packet (inp, pktlen);
break;
}
return rc;
}
int main(int argc, char **argv){
struct cb_database_r *db;
struct fasta_seq_gen *fsg;
struct fasta_seq *seq;
struct timeval start;
struct fasta_seq **coarse_sequences;
struct cb_link_to_coarse *link;
struct opt_config *conf = load_compress_args();
struct cb_compressed_seq **compressed;
uint64_t last_end, num_coarse_sequences = 0;
int i, org_seq_id, overlap;
char *fasta_filename, *compressed_filename;
FILE *compressed_file;
struct opt_args *args = opt_config_parse(conf, argc, argv);
if (args->nargs < 2) {
fprintf(stderr, "Usage: %s [flags] database-dir output-fasta-file\n",
argv[0]);
exit(1);
}
db = cb_database_r_init(args->args[0], false, false, false, 30000);
org_seq_id = 0;
gettimeofday(&start, NULL);
fasta_filename = path_join(args->args[0], CABLAST_COARSE_FASTA);
compressed_filename = path_join(args->args[0], CABLAST_COMPRESSED);
if (NULL == (compressed_file = fopen(compressed_filename, "r"))) {
fprintf(stderr, "fopen: 'fopen %s' failed: %s\n",
compressed_filename, strerror(errno));
exit(1);
}
compressed = read_compressed(compressed_file);
coarse_sequences = malloc(10000*sizeof(*coarse_sequences));
assert(coarse_sequences);
fsg = fasta_generator_start(fasta_filename, "", 100);
while (NULL != (seq = fasta_generator_next(fsg)))
coarse_sequences[num_coarse_sequences++] = seq;
for (i = 0; compressed[i] != NULL; i++) {
int current_chunk = 0;
last_end = 0;
printf(">%s\n", compressed[i]->name);
for (link = (compressed[i])->links; link != NULL; link = link->next) {
struct cb_seq *chunk =
cb_seq_init_range(-1, "",
coarse_sequences[link->coarse_seq_id]->seq,
link->coarse_start, link->coarse_end + 1);
int length;
char *decompressed;
/*overlap represents the length of the overlap of the parts of the
decompressed sequence that has been printed and the parts of the
decompressed sequence currently being decompressed.*/
overlap = last_end - link->original_start;
for (length = 0; chunk->residues[length] != '\0'; length++);
decompressed = read_edit_script(link->diff,chunk->residues,length);
/*Print all characters of the decompressed chunk past the index
*"overlap" unless overlap is greater than the length of the
*decompressed chunk.
*/
decompressed += overlap;
if (overlap < link->original_end - link->original_start ||
(overlap == link->original_end - link->original_start &&
!link->next))
printf("%s", decompressed);
decompressed -= overlap;
free(decompressed);
if (link->original_end > last_end)
last_end = link->original_end + 1;
cb_seq_free(chunk);
current_chunk++;
}
putc('\n', stdout);
}
free(fasta_filename);
free(compressed_filename);
for (i = 0; compressed[i] != NULL; i++)
cb_compressed_seq_free(compressed[i]);
free(compressed);
fasta_generator_free(fsg);
for (i = 0; i < num_coarse_sequences; i++)
free(coarse_sequences[i]);
free(coarse_sequences);
cb_database_r_free(db);
opt_config_free(conf);
opt_args_free(args);
return 0;
}
