int find_and_write_block(FILE * bf, struct file_header * fh, struct record_t * record, struct block_t * current_block){
int record_size = (int) record->size;
int free_space, res;
int i = 1;
free_space = BLOCK_SIZE - current_block->space_occupied;
if(free_space < record_size){
res = write_to_file(bf, fh, current_block);
}
/* Busco el siguiente bloque disponible o creo uno nuevo*/
while(free_space < record_size){
res = read_from_file(bf, current_block, fh, i);
if(res == -5){
initialize_block(current_block, fh->total_blocks + 1);
}
free_space = BLOCK_SIZE - current_block->space_occupied;
i++;
}
/* Escribo el bloque y actualizo los datos de control */
res = write_block(current_block, fh, record);
return res;
}
int load_data(char *filename, char *entidad){
FILE * entity_file;
FILE * block_file;
struct record_t *record;
struct file_header * file_header;
struct block_t * current_block;
char * buffer;
int res;
/* Entity file */
entity_file = open_file_to_parse(filename);
if (!entity_file){
return BAD_NAME_FILE;
}
/* Block and header file */
block_file = open_block_file(entidad, 1);
if(!block_file){
return BAD_NAME_FILE;
}else{
file_header = (struct file_header*) malloc(sizeof(struct file_header));
if(!file_header)
return ALLOCATE_FAIL;
else{
initialize_file_header(file_header);
write_header(block_file, file_header);
}
}
/* Allocate buffer and record */
buffer = (char*) malloc (RECORD_MAX_SIZE);
record = (struct record_t*) malloc(sizeof(struct record_t));
current_block = (struct block_t *) malloc(sizeof(struct block_t));
if(!buffer || !record || !current_block)
return ALLOCATE_FAIL;
initialize_buffer(buffer);
initialize_block(current_block, 1);
initialize_record(record);
/* Read text file */
while(fgets(buffer, RECORD_MAX_SIZE, entity_file) != NULL){
res = parse_line(buffer, record, RECORD_MAX_SIZE);
if(res == RES_OK){
res = find_and_write_block(block_file, file_header, record, current_block);
initialize_record(record);
free(record->content);
};
};
write_to_file(block_file, file_header, current_block);
print_header(file_header);
free(record);
free(buffer);
free(file_header);
free(current_block);
close_entity_file(entity_file);
close_block_file(block_file);
return res;
}
int search_by_code(char* entidad, int code){
FILE * block_file;
struct record_t *record;
struct block_t * current_block;
struct file_header * file_header;
int res, i = 1, j = 1, recs = 0;
int current_pos = 0, encontrado = 1;
block_file = open_block_file(entidad, 2);
if(!block_file){
return BAD_NAME_FILE;
}
/* Allocate buffer and record */
record = (struct record_t*) malloc(sizeof(struct record_t));
current_block = (struct block_t *) malloc(sizeof(struct block_t));
file_header = (struct file_header*) malloc(sizeof(struct file_header));
if(!record || !current_block || !file_header)
return ALLOCATE_FAIL;
initialize_record(record);
initialize_block(current_block,1);
read_header(block_file, file_header);
while((i < file_header->total_blocks+1) && (encontrado == 1)){
current_pos = 0;
j = 1;
res = read_from_file(block_file, current_block, file_header, i);
if(res == RES_OK){
recs = current_block->total_reg;
while((j < recs+1) && (encontrado == 1)){
initialize_record(record);
current_pos += read_block(current_block, record, current_pos);
if(code == record->code){
print_success_message(record);
encontrado = 0;
}
free(record->content);
j++;
}
}
i++;
}
if(encontrado == 1){
print_error_message(code);
}
free(record);
free(file_header);
free(current_block);
fclose(block_file);
return res;
}
int main(int argc, char* argv[]){
initialize_block(argc,argv,sizeof(struct global_settings),1,0);
start_task(gs->task_prio,&loop);
wait_for_task_end();
finalize_block();
return 0;
}
// Method to process a received packet
// Processing involves the following
// (1) header extraction
// (2) block population (output buffer data type is a block)
// (3) buffer population (if block is filled)
static inline uint64_t process_packet(flag_input_databuf_t * db, struct hashpipe_udp_packet *p) {
static block_info_t binfo;
packet_header_t pkt_header;
// Parse packet header
get_header(p, &pkt_header);
uint64_t pkt_mcnt = pkt_header.mcnt;
uint64_t cur_mcnt = binfo.mcnt_start;
int dest_block_idx = get_block_idx(pkt_mcnt);
// fprintf(stdout, "Block idx = %d\n", dest_block_idx);
// Check mcnt to see if packet belongs in current block, next, or the one after
uint64_t pkt_mcnt_dist = pkt_mcnt - cur_mcnt;
if (pkt_mcnt_dist >= 2*Nm) { // 2nd next block (Current block + 2)
set_block_filled(db, &binfo);
// Advance mcnt_start to next block
cur_mcnt += Nm;
binfo.mcnt_start += Nm;
binfo.block_i = (binfo.block_i + 1) % N_INPUT_BLOCKS;
// Initialize next block
flag_input_databuf_wait_free(db, dest_block_idx);
initialize_block(db, pkt_mcnt);
// Reset packet counter for this block
binfo.packet_count[dest_block_idx] = 0;
}
// Increment packet count for block
binfo.packet_count[dest_block_idx]++;
// Validate FID and XID
// Calculate "m" and "f" which index the buffer for writing packet payload
if (calc_block_indices(&binfo, &pkt_header) == -1) {
hashpipe_error(__FUNCTION__, "invalid FID and XID in header");
return -1;
}
// Calculate starting points for writing packet payload into buffer
// POSSIBLE RACE CONDITION!!!! Need to lock db->block access with semaphore
uint64_t * dest_p = db->block[dest_block_idx].data + flag_input_databuf_idx(binfo.m, binfo.f, 0, 0);
const uint64_t * payload_p = (uint64_t *)(p->data+8);
// Copy data into buffer
memcpy(dest_p, payload_p, N_BYTES_PER_PACKET);
// print_pkt_header(&pkt_header);
return pkt_mcnt;
}
int list_data(char *filename){
FILE * block_file;
struct record_t *record;
struct block_t * current_block;
struct file_header * file_header;
int res, i, j, recs, k;
int current_pos = 0;
block_file = open_block_file(filename, 2);
if(!block_file)
return BAD_NAME_FILE;
/* Allocate buffer and record */
record = (struct record_t*) malloc(sizeof(struct record_t));
current_block = (struct block_t *) malloc(sizeof(struct block_t));
file_header = (struct file_header*) malloc(sizeof(struct file_header));
if(!record || !current_block || !file_header)
return ALLOCATE_FAIL;
initialize_record(record);
initialize_block(current_block,1);
read_header(block_file, file_header);
printf("Listado de %s\n\n", filename);
k=0;
for(i=1;i<file_header->total_blocks+1; i++){
current_pos = 0;
res = read_from_file(block_file, current_block, file_header, i);
if(res == RES_OK){
recs = current_block->total_reg;
for(j=0; j < recs; j++){
initialize_record(record);
current_pos += read_block(current_block, record, current_pos);
printf("%03d) %d, %s\n",k+1, record->code, record->content);
k++;
free(record->content);
}
}
}
printf("\n");
free(record);
free(file_header);
free(current_block);
fclose(block_file);
return res;
}
int order(char * entidad, int (*fp)(struct record_t*, struct record_t*)){
FILE * block_file;
FILE * tmp_file;
struct record_t *record;
struct block_t * current_block;
struct file_header * file_header;
int res, j = 0;
int current_pos = 0;
int current_pos_freeze = 0;
int registros_leidos = 0, bloques_leidos = 0;
int todo_leido = 1, totales_leidos = 0;
int registro_en_memoria = 1, registro_procesado = 0;
int tmp_files = 1;
char path[30];
struct record_t buffer[TMP_BUFFER];
struct record_t freeze_buffer[TMP_BUFFER];
block_file = open_block_file(entidad, 2);
if(!block_file){
return BAD_NAME_FILE;
}
/* Allocate buffer and record */
current_block = (struct block_t *) malloc(sizeof(struct block_t));
file_header = (struct file_header*) malloc(sizeof(struct file_header));
record = (struct record_t*) malloc(sizeof(struct record_t));
if(!current_block || !file_header || !record)
return ALLOCATE_FAIL;
initialize_block(current_block,1);
read_header(block_file, file_header);
initialize_record(record);
res = read_from_file(block_file, current_block, file_header, 1);
bloques_leidos++;
if(res == RES_OK){
/* Inicializo el buffer con registros */
while(j<TMP_BUFFER && (todo_leido == 1)){
if(registros_leidos < current_block->total_reg){
current_pos += read_block(current_block, record, current_pos);
memcpy(&buffer[j],record,sizeof(struct record_t));
registros_leidos++;
totales_leidos++;
j++;
}else{
if((bloques_leidos < file_header->total_blocks+1) && (file_header->total_records > totales_leidos)){
res = read_from_file(block_file, current_block, file_header, current_block->number+1);
bloques_leidos++;
registros_leidos = 0;
current_pos = 0;
}else{
todo_leido = 0;
}
}
}
/* Creo el primer sort file */
sprintf(path,"tmp/sort%d.txt",tmp_files);
tmp_file = fopen(path, "w");
if(file_header->total_records == totales_leidos)
heapify(buffer,totales_leidos, fp);
else
heapify(buffer,TMP_BUFFER, fp);
while((todo_leido == 1) && tmp_file){
if(registros_leidos < current_block->total_reg){
current_pos += read_block(current_block, record, current_pos);
registros_leidos++;
registro_en_memoria = 0;
registro_procesado = 1;
}else{
if(bloques_leidos < file_header->total_blocks){
res = read_from_file(block_file, current_block, file_header, current_block->number+1);
bloques_leidos++;
current_pos = 0;
registros_leidos = 0;
registro_procesado = 1;
registro_en_memoria = 1;
}else{
todo_leido = 0;
}
}
while((registro_procesado == 1) && (registro_en_memoria == 0)){
if(fp(record,&buffer[0]) < 0){
if(current_pos_freeze<TMP_BUFFER){
memcpy(&freeze_buffer[current_pos_freeze],record,sizeof(struct record_t));
current_pos_freeze++;
registro_procesado = 0;
registro_en_memoria = 1;
}
else {
/* vacio el buffer en el archivo */
print_buffer_to_file(buffer, TMP_BUFFER, tmp_file, fp);
/* cierro archivo y abro el siguiente */
tmp_files++;
fclose(tmp_file);
sprintf(path,"tmp/sort%d.txt",tmp_files);
tmp_file = fopen(path, "w");
/* copio el freeze_buffer al buffer y creo el heap*/
heapify(freeze_buffer,TMP_BUFFER, fp);
memcpy(buffer,freeze_buffer,sizeof(struct record_t) * TMP_BUFFER);
current_pos_freeze = 0;
registro_procesado = 1;
}
}
else{
/* Guardo en el archivo, libero memoria */
print_record_to_file(buffer, tmp_file);
/* Guardo en el buffer el record leido */
memcpy(&buffer[0],&buffer[TMP_BUFFER-1],sizeof(struct record_t));
memcpy(&buffer[TMP_BUFFER-1],record,sizeof(struct record_t));
registro_procesado = 0;
registro_en_memoria = 1;
update(buffer, TMP_BUFFER, fp);
}
}
}
/* Guardo en el archivo*/
if((file_header->total_records == totales_leidos) && (current_block->number == 1))
print_buffer_to_file(buffer, totales_leidos, tmp_file, fp);
else
print_buffer_to_file(buffer, TMP_BUFFER, tmp_file, fp);
fclose(tmp_file);
/* Guardo lo que tenia en el freeze_buffer en otro archivo */
if(current_pos_freeze > 1){
tmp_files++;
sprintf(path,"tmp/sort%d.txt",tmp_files);
tmp_file = fopen(path, "w");
print_buffer_to_file(freeze_buffer, current_pos_freeze, tmp_file, fp);
fclose(tmp_file);
}
}
free(record);
free(current_block);
free(file_header);
fclose(block_file);
/* Merge de los archivos generados */
res = merge_files(tmp_files, fp);
printf("\n");
return res;
}
// Run method for the thread
// It is meant to do the following:
// (1) Initialize status buffer
// (2) Set up network parameters and socket
// (3) Start main loop
// (3a) Receive packet on socket
// (3b) Error check packet (packet size, etc)
// (3c) Call process_packet on received packet
// (4) Terminate thread cleanly
static void *run(hashpipe_thread_args_t * args) {
fprintf(stdout, "N_INPUTS = %d\n", N_INPUTS);
fprintf(stdout, "N_CHAN = %d\n", N_CHAN);
fprintf(stdout, "N_CHAN_PER_X = %d\n", N_CHAN_PER_X);
fprintf(stdout, "N_CHAN_PER_PACKET = %d\n", N_CHAN_PER_PACKET);
fprintf(stdout, "N_TIME_PER_PACKET = %d\n", N_TIME_PER_PACKET);
fprintf(stdout, "N_TIME_PER_BLOCK = %d\n", N_TIME_PER_BLOCK);
fprintf(stdout, "N_BYTES_PER_BLOCK = %d\n", N_BYTES_PER_BLOCK);
fprintf(stdout, "N_BYTES_PER_PACKET = %d\n", N_BYTES_PER_PACKET);
fprintf(stdout, "N_PACKETS_PER_BLOCK = %d\n", N_PACKETS_PER_BLOCK);
fprintf(stdout, "N_COR_MATRIX = %d\n", N_COR_MATRIX);
// Local aliases to shorten access to args fields
// Our output buffer happens to be a paper_input_databuf
flag_input_databuf_t *db = (flag_input_databuf_t *)args->obuf;
hashpipe_status_t st = args->st;
const char * status_key = args->thread_desc->skey;
st_p = &st; // allow global (this source file) access to the status buffer
/* Read network params */
fprintf(stdout, "Setting up network parameters\n");
struct hashpipe_udp_params up = {
.bindhost = "0.0.0.0",
.bindport = 8511,
.packet_size = 8008
};
hashpipe_status_lock_safe(&st);
// Get info from status buffer if present (no change if not present)
hgets(st.buf, "BINDHOST", 80, up.bindhost);
hgeti4(st.buf, "BINDPORT", &up.bindport);
// Store bind host/port info etc in status buffer
hputs(st.buf, "BINDHOST", up.bindhost);
hputi4(st.buf, "BINDPORT", up.bindport);
hputu4(st.buf, "MISSEDFE", 0);
hputu4(st.buf, "MISSEDPK", 0);
hputs(st.buf, status_key, "running");
hashpipe_status_unlock_safe(&st);
struct hashpipe_udp_packet p;
/* Give all the threads a chance to start before opening network socket */
int netready = 0;
int corready = 0;
int checkready = 0;
while (!netready) {
sleep(1);
// Check the correlator to see if it's ready yet
hashpipe_status_lock_safe(&st);
hgeti4(st.buf, "CORREADY", &corready);
hgeti4(st.buf, "SAVEREADY", &checkready);
hashpipe_status_unlock_safe(&st);
if (!corready) {
continue;
}
//if (!checkready) {
// continue;
//}
// Check the other threads to see if they're ready yet
// TBD
// If we get here, then all threads are initialized
netready = 1;
}
sleep(3);
/* Set up UDP socket */
fprintf(stderr, "NET: BINDHOST = %s\n", up.bindhost);
fprintf(stderr, "NET: BINDPORT = %d\n", up.bindport);
int rv = hashpipe_udp_init(&up);
if (rv!=HASHPIPE_OK) {
hashpipe_error("paper_net_thread",
"Error opening UDP socket.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)hashpipe_udp_close, &up);
// Initialize first few blocks in the buffer
int i;
for (i = 0; i < 2; i++) {
// Wait until block semaphore is free
if (flag_input_databuf_wait_free(db, i) != HASHPIPE_OK) {
if (errno == EINTR) { // Interrupt occurred
hashpipe_error(__FUNCTION__, "waiting for free block interrupted\n");
pthread_exit(NULL);
} else {
hashpipe_error(__FUNCTION__, "error waiting for free block\n");
pthread_exit(NULL);
}
}
initialize_block(db, i*Nm);
}
// Set correlator to "start" state
hashpipe_status_lock_safe(&st);
hputs(st.buf, "INTSTAT", "start");
hashpipe_status_unlock_safe(&st);
/* Main loop */
uint64_t packet_count = 0;
fprintf(stdout, "Net: Starting Thread!\n");
while (run_threads()) {
// Get packet
do {
p.packet_size = recv(up.sock, p.data, HASHPIPE_MAX_PACKET_SIZE, 0);
} while (p.packet_size == -1 && (errno == EAGAIN || errno == EWOULDBLOCK) && run_threads());
if(!run_threads()) break;
if (up.packet_size != p.packet_size && up.packet_size != p.packet_size-8) {
// If an error was returned instead of a valid packet size
if (p.packet_size == -1) {
fprintf(stderr, "uh oh!\n");
// Log error and exit
hashpipe_error("paper_net_thread",
"hashpipe_udp_recv returned error");
perror("hashpipe_udp_recv");
pthread_exit(NULL);
} else {
// Log warning and ignore wrongly sized packet
hashpipe_warn("paper_net_thread", "Incorrect pkt size (%d)", p.packet_size);
continue;
}
}
packet_count++;
process_packet(db, &p);
/* Will exit if thread has been cancelled */
pthread_testcancel();
}
pthread_cleanup_pop(1); /* Closes push(hashpipe_udp_close) */
hashpipe_status_lock_busywait_safe(&st);
hputs(st.buf, status_key, "terminated");
hashpipe_status_unlock_safe(&st);
return NULL;
}
static hashpipe_thread_desc_t net_thread = {
name: "flag_net_thread",
skey: "NETSTAT",
init: NULL,
run: run,
ibuf_desc: {NULL},
// This function returns -1 unless the given packet causes a block to be marked
// as filled in which case this function returns the marked block's first mcnt.
// Any return value other than -1 will be stored in the status memory as
// NETMCNT, so it is important that values other than -1 are returned rarely
// (i.e. when marking a block as filled)!!!
static inline uint64_t write_paper_packet_to_blocks(paper_input_databuf_t *paper_input_databuf_p, struct guppi_udp_packet *p) {
static block_info_t binfo;
packet_header_t pkt_header;
const uint64_t *payload_p;
int rv;
int i;
uint64_t *dest_p;
uint64_t netmcnt = -1; // Value to store in status memory
#if N_DEBUG_INPUT_BLOCKS == 1
static uint64_t debug_remaining = -1ULL;
static off_t debug_offset = 0;
uint64_t * debug_ptr;
#endif
// housekeeping for each packet
get_header(p, &pkt_header);
#if N_DEBUG_INPUT_BLOCKS == 1
debug_ptr = (uint64_t *)&paper_input_databuf_p->block[N_INPUT_BLOCKS];
debug_ptr[debug_offset++] = be64toh(*(uint64_t *)(p->data));
if(--debug_remaining == 0) {
exit(1);
}
if(debug_offset >= sizeof(paper_input_block_t)/sizeof(uint64_t)) {
debug_offset = 0;
}
#endif
if(! binfo.initialized) {
// insist that we start on a multiple of sub_blocks/block
if(pkt_header.mcnt % N_SUB_BLOCKS_PER_INPUT_BLOCK != 0) {
return -1;
}
initialize_block_info(paper_input_databuf_p, &binfo, pkt_header.mcnt);
}
if(out_of_seq_mcnt(&binfo, pkt_header.mcnt)) {
return(handle_out_of_seq_mcnt(&binfo));
}
if((rv = calc_block_indexes(&binfo, &pkt_header))) {
return rv;
}
if(! binfo.block_active[binfo.block_i]) {
// new block, pass along the block for N_INPUT_BLOCKS/2 blocks ago
i = subtract_block_i(binfo.block_i, N_INPUT_BLOCKS/2);
#if N_DEBUG_INPUT_BLOCKS == 1
#define DEBUG_EXTRA (0x10000)
if(set_block_filled(paper_input_databuf_p, &binfo, i) && debug_remaining > DEBUG_EXTRA) {
debug_remaining = DEBUG_EXTRA;
}
#else
set_block_filled(paper_input_databuf_p, &binfo, i);
#endif
netmcnt = paper_input_databuf_p->block[i].header.mcnt;
// Wait (hopefully not long!) for free block for this packet
if((rv = paper_input_databuf_busywait_free(paper_input_databuf_p, binfo.block_i)) != GUPPI_OK) {
if (rv==GUPPI_TIMEOUT) {
// run_threads is 0 (i.e. shutting down)
return -1;
} else {
guppi_error(__FUNCTION__, "error waiting for free databuf");
run_threads=0;
pthread_exit(NULL);
return -1;
}
}
initialize_block(paper_input_databuf_p, &binfo, pkt_header.mcnt);
}
binfo.block_active[binfo.block_i]++; // increment packet count for block
// end housekeeping
// Calculate starting points for unpacking this packet into block's data buffer.
dest_p = paper_input_databuf_p->block[binfo.block_i].data
+ paper_input_databuf_data_idx(binfo.m, binfo.x, binfo.q, binfo.f, 0, 0);
payload_p = (uint64_t *)(p->data+8);
// Copy data into buffer
memcpy(dest_p, payload_p, N_BYTES_PER_PACKET);
return netmcnt;
}
int main(int argc, char **argv){
int i=0, res, free_space;
char content[17] = "Esto es un test ";
FILE * block_file;
struct file_header * fh;
struct block_t * block;
struct record_t * record;
printf("Welcome to Carga Masiva Test\n\n");
block_file = fopen("block_records_test.dat", "w");
fh = (struct file_header*) malloc(sizeof(struct file_header));
record = (struct record_t*) malloc(sizeof(struct record_t));
block = (struct block_t *) malloc(sizeof(struct block_t));
if(!record || !block || !fh)
return -1;
initialize_block(block, 1);
initialize_record(record);
initialize_file_header(fh);
res = write_header(block_file, fh);
if(res == 0){
for(i=0;i<50;i++){
record->content = (char*) malloc(strlen(content)+1);
memcpy(record->content, content, strlen(content)+1);
record->content[17] = '\0';
record->code = i+1;
record->size = (int)sizeof(record->size) + (int)sizeof(record->code) + (int) strlen(record->content) + 1;
free_space = MAX_SIZE - block->space_occupied;
if(free_space < record->size){
write_to_file(block_file, fh, block);
printf("(Bloque %d) \tOcupado: %d, Records: %d\n", block->number, block->space_occupied, block->total_reg);
initialize_block(block, block->number+1);
}
write_block(block, fh, record);
initialize_record(record);
free(record->content);
}
}
write_to_file(block_file, fh, block);
printf("(Bloque %d) \tOcupado: %d, Records: %d\n", block->number, block->space_occupied, block->total_reg);
printf("\nListo la carga de bloques\n");
printf("---------------------------------------\n");
print_header(fh);
free(block);
free(record);
free(fh);
fclose(block_file);
return 0;
}
