static node * alloc_blocks_for_file(const char * fn)
{
int bytes;
FILE * f;
node * n = new_node();
f = fopen(fn, "r");
if (f == NULL) return NULL;
fseek(f, 0, SEEK_END);
bytes = ftell(f);
fclose(f);
if (bytes % 0x1000 == 0) {
n->u.data.block_count = bytes / 0x1000;
n->u.data.bytes_in_last_block = 0x1000;
} else {
n->u.data.block_count = bytes / 0x1000 + 1;
n->u.data.bytes_in_last_block = bytes % 0x1000;
}
if (n->u.data.block_count > 0xffffff) {
// FIXME
error("Maximium block size exceeded by file '%s'\n", fn);
}
n->type = NT_DATA;
n->u.data.next = NULL;
n->u.data.lba = alloc_blocks(n->u.data.block_count);
return n;
}
void append_data_to_file(int inum, u8 *data, size_t size)
{
printf("append data file at num %i\n", inum);
inode *in = inode_at_num(inum);
printf("inode: num: %i, size: %i, blocks: %i, end addr: %i\n",
in->num, in->size, in->blocks, in->end_block);
block *cur_blk = block_from_num(in->end_block);
printf("block addr: %i\n", in->end_block);
int pos = in->size % BLOCK_SIZE;
int blocks = (pos + size) / BLOCK_SIZE;
int numbytes = BLOCK_SIZE - pos;
write_data_to_block(cur_blk, data, size, pos);
in->size += numbytes;
//printf("%i:",pos);
for (int i = 0; i < blocks; i++) {
alloc_blocks(in, 1);
cur_blk = block_from_num(in->end_block);
int pos = in->size % BLOCK_SIZE;
int numbytes = BLOCK_SIZE - pos;
write_data_to_block(cur_blk, data, numbytes, pos);
in->size += numbytes;
}
write_block_to_disk(cur_blk);
in->size += size;
write_inode_to_disk(in);
free(in);
free(cur_blk);
}
/**
* This reads data from the given file pointer into memory, breaking it into segments
* of the given number of lines, to ease memory management issues at the cost of some
* overhead. This assumes that the file consists entirely of quality scores with no
* other lines in between
* @param path Path of the file to read
* @param info Information structure to store in, this must be a valid pointer already
* @param max_lines Maximum number of lines to read, will override the actual number in the file if >0
*/
uint32_t load_file(const char *path, struct quality_file_t *info, uint64_t max_lines) {
uint32_t status, block_idx, line_idx;
uint32_t j;
char line[READ_LINEBUF_LENGTH];
FILE *fp;
struct _stat finfo;
// Load metadata into the info structure
info->path = strdup(path);
fp = fopen(path, "rt");
if (!fp) {
return LF_ERROR_NOT_FOUND;
}
// Use the first line to figure out how long the file is
fgets(line, READ_LINEBUF_LENGTH, fp);
info->columns = strlen(line) - 1;
if (info->columns > MAX_READS_PER_LINE) {
fclose(fp);
return LF_ERROR_TOO_LONG;
}
// Figure out how many lines we'll need depending on whether we were limited or not
_stat(path, &finfo);
info->lines = finfo.st_size / ((uint64_t) (info->columns+1));
if (max_lines > 0 && info->lines > max_lines) {
info->lines = max_lines;
}
status = alloc_blocks(info);
if (status != LF_ERROR_NONE)
return status;
// Process the file
block_idx = 0;
line_idx = 0;
fseek(fp, 0, SEEK_SET);
while (!feof(fp) && (block_idx * MAX_LINES_PER_BLOCK + line_idx) < info->lines) {
// Read line and store in our array with data conversion, also stripping newlines
fgets(line, READ_LINEBUF_LENGTH, fp);
for (j = 0; j < info->columns; ++j) {
info->blocks[block_idx].lines[line_idx].data[j] = line[j] - 33;
}
// Increment line/block pointers as necesary
line_idx += 1;
if (line_idx == info->blocks[block_idx].count) {
line_idx = 0;
block_idx += 1;
}
}
fclose(fp);
return LF_ERROR_NONE;
}
/**
* This reads data from the given file pointer into memory, breaking it into segments
* of the given number of lines, to ease memory management issues at the cost of some
* overhead. This assumes that the file consists entirely of quality scores with no
* other lines in between
* @param path Path of the file to read
* @param info Information structure to store in, this must be a valid pointer already
* @param max_lines Maximum number of lines to read, will override the actual number in the file if >0
* @todo @xxx This assumes we have only newlines in the file despite some vague attempts to handle \r\n as well
* @todo @xxx It WILL break the mapping if given a file with \r characters
* @todo Implement windows analog to mmap to provide the same facility
*/
uint32_t generate_qv_struct(struct qv_file_t* qvs, struct quality_file_t *info, uint64_t max_lines) {
uint32_t status, block_idx, line_idx;
//figure out how long the file is
info->columns = qvs->read_length;
if (info->columns > MAX_READS_PER_LINE) {
return LF_ERROR_TOO_LONG;
}
// Figure out how many lines we'll need depending on whether we were limited or not
info->lines = qvs->lines;
if (max_lines > 0 && info->lines > max_lines) {
info->lines = max_lines;
}
status = alloc_blocks(info);
if (status != LF_ERROR_NONE)
return status;
// Process the file
block_idx = 0;
line_idx = 0;
while ((block_idx * MAX_LINES_PER_BLOCK + line_idx) < info->lines) {
// Setting up mmap indexing assumes we have only one line ending!
info->blocks[block_idx].lines[line_idx].m_data = qvs->file_head + ((uint64_t) (block_idx * MAX_LINES_PER_BLOCK + line_idx)) * (info->columns+1);
// Increment line/block pointers as necesary
line_idx += 1;
if (line_idx == info->blocks[block_idx].count) {
line_idx = 0;
block_idx += 1;
printf("block %d loaded\n", block_idx);
}
}
return LF_ERROR_NONE;
}
static node * new_node(void)
{
node * n = malloc(sizeof(node));
if (file_sys.cur_tbl_entry >= 0x100) {
file_sys.cur_tbl_entry = 1;
file_sys.cur_tbl_lba = alloc_blocks(2);
file_sys.cur_high_addr += 0x100;
if (file_sys.mbl == NULL) {
file_sys.mbl = malloc(sizeof(meta_block_list));
} else {
file_sys.mbl->next = malloc(sizeof(meta_block_list));
file_sys.mbl = file_sys.mbl->next;
}
file_sys.mbl->lba = file_sys.cur_tbl_lba;
}
n->node_ptr = file_sys.cur_high_addr | file_sys.cur_tbl_entry;
file_sys.cur_tbl_entry++;
return n;
}
/**
* This reads data from the given file pointer into memory, breaking it into segments
* of the given number of lines, to ease memory management issues at the cost of some
* overhead. This assumes that the file consists entirely of quality scores with no
* other lines in between
* @param path Path of the file to read
* @param info Information structure to store in, this must be a valid pointer already
* @param max_lines Maximum number of lines to read, will override the actual number in the file if >0
* @todo @xxx This assumes we have only newlines in the file despite some vague attempts to handle \r\n as well
* @todo @xxx It WILL break the mapping if given a file with \r characters
* @todo Implement windows analog to mmap to provide the same facility
*/
uint32_t load_qv_file(const char *path, struct quality_file_t *info, uint64_t max_lines) {
uint32_t status, block_idx, line_idx;
char line[READ_LINEBUF_LENGTH];
FILE *fp;
int fd;
struct _stat finfo;
void *file_mmap;
// Load metadata into the info structure
info->path = strdup(path);
fp = fopen(path, "rt");
fd = open(path, O_RDONLY);
if (!fp || fd == -1) {
return LF_ERROR_NOT_FOUND;
}
// Use the first line to figure out how long the file is
fgets(line, READ_LINEBUF_LENGTH, fp);
info->columns = strlen(line) - 1;
if (info->columns > MAX_READS_PER_LINE) {
fclose(fp);
return LF_ERROR_TOO_LONG;
}
fclose(fp);
// Figure out how many lines we'll need depending on whether we were limited or not
_stat(path, &finfo);
info->lines = finfo.st_size / ((uint64_t) (info->columns+1));
fp = fopen(path, "rt");
int ch, number_of_lines = 0, read_length = 0;
do
{
ch = fgetc(fp);
read_length++;
if(ch == '\n')
number_of_lines++;
} while (ch != EOF);
fclose(fp);
info->lines = number_of_lines;
if (max_lines > 0 && info->lines > max_lines) {
info->lines = max_lines;
}
status = alloc_blocks(info);
if (status != LF_ERROR_NONE)
return status;
// mmap the file to set up constant pointers indexing it
file_mmap = mmap(NULL, finfo.st_size, PROT_READ, MAP_SHARED, fd, 0);
// Process the file
block_idx = 0;
line_idx = 0;
fp = fopen(path, "r");
read_length = 0;
long long unsigned int cumm_pos = 0;
unsigned int max_read_length = 0;
while ((block_idx * MAX_LINES_PER_BLOCK + line_idx) < info->lines) {
info->blocks[block_idx].lines[line_idx].m_data = file_mmap + cumm_pos;
while((ch = fgetc(fp)) != '\n'){
read_length++;
if(read_length>max_read_length)max_read_length = read_length;
cumm_pos++;
}
cumm_pos++;
info->blocks[block_idx].lines[line_idx].line_length = read_length;
read_length = 0;
// Setting up mmap indexing assumes we have only one line ending!
//info->blocks[block_idx].lines[line_idx].m_data = file_mmap + ((uint64_t) (block_idx * MAX_LINES_PER_BLOCK + line_idx)) * (info->columns+1);
// Increment line/block pointers as necesary
line_idx += 1;
if (line_idx == info->blocks[block_idx].count) {
line_idx = 0;
block_idx += 1;
}
}
info->columns = max_read_length;
return LF_ERROR_NONE;
}
inline T * alloc_blocks (size_t block_count)
{
return static_cast<T *>(alloc_blocks(sizeof(T), block_count));
}