int find_and_write_block(FILE * bf, struct file_header * fh, struct record_t * record, struct block_t * current_block){ int record_size = (int) record->size; int free_space, res; int i = 1; free_space = BLOCK_SIZE - current_block->space_occupied; if(free_space < record_size){ res = write_to_file(bf, fh, current_block); } /* Busco el siguiente bloque disponible o creo uno nuevo*/ while(free_space < record_size){ res = read_from_file(bf, current_block, fh, i); if(res == -5){ initialize_block(current_block, fh->total_blocks + 1); } free_space = BLOCK_SIZE - current_block->space_occupied; i++; } /* Escribo el bloque y actualizo los datos de control */ res = write_block(current_block, fh, record); return res; }
int load_data(char *filename, char *entidad){ FILE * entity_file; FILE * block_file; struct record_t *record; struct file_header * file_header; struct block_t * current_block; char * buffer; int res; /* Entity file */ entity_file = open_file_to_parse(filename); if (!entity_file){ return BAD_NAME_FILE; } /* Block and header file */ block_file = open_block_file(entidad, 1); if(!block_file){ return BAD_NAME_FILE; }else{ file_header = (struct file_header*) malloc(sizeof(struct file_header)); if(!file_header) return ALLOCATE_FAIL; else{ initialize_file_header(file_header); write_header(block_file, file_header); } } /* Allocate buffer and record */ buffer = (char*) malloc (RECORD_MAX_SIZE); record = (struct record_t*) malloc(sizeof(struct record_t)); current_block = (struct block_t *) malloc(sizeof(struct block_t)); if(!buffer || !record || !current_block) return ALLOCATE_FAIL; initialize_buffer(buffer); initialize_block(current_block, 1); initialize_record(record); /* Read text file */ while(fgets(buffer, RECORD_MAX_SIZE, entity_file) != NULL){ res = parse_line(buffer, record, RECORD_MAX_SIZE); if(res == RES_OK){ res = find_and_write_block(block_file, file_header, record, current_block); initialize_record(record); free(record->content); }; }; write_to_file(block_file, file_header, current_block); print_header(file_header); free(record); free(buffer); free(file_header); free(current_block); close_entity_file(entity_file); close_block_file(block_file); return res; }
int search_by_code(char* entidad, int code){ FILE * block_file; struct record_t *record; struct block_t * current_block; struct file_header * file_header; int res, i = 1, j = 1, recs = 0; int current_pos = 0, encontrado = 1; block_file = open_block_file(entidad, 2); if(!block_file){ return BAD_NAME_FILE; } /* Allocate buffer and record */ record = (struct record_t*) malloc(sizeof(struct record_t)); current_block = (struct block_t *) malloc(sizeof(struct block_t)); file_header = (struct file_header*) malloc(sizeof(struct file_header)); if(!record || !current_block || !file_header) return ALLOCATE_FAIL; initialize_record(record); initialize_block(current_block,1); read_header(block_file, file_header); while((i < file_header->total_blocks+1) && (encontrado == 1)){ current_pos = 0; j = 1; res = read_from_file(block_file, current_block, file_header, i); if(res == RES_OK){ recs = current_block->total_reg; while((j < recs+1) && (encontrado == 1)){ initialize_record(record); current_pos += read_block(current_block, record, current_pos); if(code == record->code){ print_success_message(record); encontrado = 0; } free(record->content); j++; } } i++; } if(encontrado == 1){ print_error_message(code); } free(record); free(file_header); free(current_block); fclose(block_file); return res; }
int main(int argc, char* argv[]){ initialize_block(argc,argv,sizeof(struct global_settings),1,0); start_task(gs->task_prio,&loop); wait_for_task_end(); finalize_block(); return 0; }
// Method to process a received packet // Processing involves the following // (1) header extraction // (2) block population (output buffer data type is a block) // (3) buffer population (if block is filled) static inline uint64_t process_packet(flag_input_databuf_t * db, struct hashpipe_udp_packet *p) { static block_info_t binfo; packet_header_t pkt_header; // Parse packet header get_header(p, &pkt_header); uint64_t pkt_mcnt = pkt_header.mcnt; uint64_t cur_mcnt = binfo.mcnt_start; int dest_block_idx = get_block_idx(pkt_mcnt); // fprintf(stdout, "Block idx = %d\n", dest_block_idx); // Check mcnt to see if packet belongs in current block, next, or the one after uint64_t pkt_mcnt_dist = pkt_mcnt - cur_mcnt; if (pkt_mcnt_dist >= 2*Nm) { // 2nd next block (Current block + 2) set_block_filled(db, &binfo); // Advance mcnt_start to next block cur_mcnt += Nm; binfo.mcnt_start += Nm; binfo.block_i = (binfo.block_i + 1) % N_INPUT_BLOCKS; // Initialize next block flag_input_databuf_wait_free(db, dest_block_idx); initialize_block(db, pkt_mcnt); // Reset packet counter for this block binfo.packet_count[dest_block_idx] = 0; } // Increment packet count for block binfo.packet_count[dest_block_idx]++; // Validate FID and XID // Calculate "m" and "f" which index the buffer for writing packet payload if (calc_block_indices(&binfo, &pkt_header) == -1) { hashpipe_error(__FUNCTION__, "invalid FID and XID in header"); return -1; } // Calculate starting points for writing packet payload into buffer // POSSIBLE RACE CONDITION!!!! Need to lock db->block access with semaphore uint64_t * dest_p = db->block[dest_block_idx].data + flag_input_databuf_idx(binfo.m, binfo.f, 0, 0); const uint64_t * payload_p = (uint64_t *)(p->data+8); // Copy data into buffer memcpy(dest_p, payload_p, N_BYTES_PER_PACKET); // print_pkt_header(&pkt_header); return pkt_mcnt; }
int list_data(char *filename){ FILE * block_file; struct record_t *record; struct block_t * current_block; struct file_header * file_header; int res, i, j, recs, k; int current_pos = 0; block_file = open_block_file(filename, 2); if(!block_file) return BAD_NAME_FILE; /* Allocate buffer and record */ record = (struct record_t*) malloc(sizeof(struct record_t)); current_block = (struct block_t *) malloc(sizeof(struct block_t)); file_header = (struct file_header*) malloc(sizeof(struct file_header)); if(!record || !current_block || !file_header) return ALLOCATE_FAIL; initialize_record(record); initialize_block(current_block,1); read_header(block_file, file_header); printf("Listado de %s\n\n", filename); k=0; for(i=1;i<file_header->total_blocks+1; i++){ current_pos = 0; res = read_from_file(block_file, current_block, file_header, i); if(res == RES_OK){ recs = current_block->total_reg; for(j=0; j < recs; j++){ initialize_record(record); current_pos += read_block(current_block, record, current_pos); printf("%03d) %d, %s\n",k+1, record->code, record->content); k++; free(record->content); } } } printf("\n"); free(record); free(file_header); free(current_block); fclose(block_file); return res; }
int order(char * entidad, int (*fp)(struct record_t*, struct record_t*)){ FILE * block_file; FILE * tmp_file; struct record_t *record; struct block_t * current_block; struct file_header * file_header; int res, j = 0; int current_pos = 0; int current_pos_freeze = 0; int registros_leidos = 0, bloques_leidos = 0; int todo_leido = 1, totales_leidos = 0; int registro_en_memoria = 1, registro_procesado = 0; int tmp_files = 1; char path[30]; struct record_t buffer[TMP_BUFFER]; struct record_t freeze_buffer[TMP_BUFFER]; block_file = open_block_file(entidad, 2); if(!block_file){ return BAD_NAME_FILE; } /* Allocate buffer and record */ current_block = (struct block_t *) malloc(sizeof(struct block_t)); file_header = (struct file_header*) malloc(sizeof(struct file_header)); record = (struct record_t*) malloc(sizeof(struct record_t)); if(!current_block || !file_header || !record) return ALLOCATE_FAIL; initialize_block(current_block,1); read_header(block_file, file_header); initialize_record(record); res = read_from_file(block_file, current_block, file_header, 1); bloques_leidos++; if(res == RES_OK){ /* Inicializo el buffer con registros */ while(j<TMP_BUFFER && (todo_leido == 1)){ if(registros_leidos < current_block->total_reg){ current_pos += read_block(current_block, record, current_pos); memcpy(&buffer[j],record,sizeof(struct record_t)); registros_leidos++; totales_leidos++; j++; }else{ if((bloques_leidos < file_header->total_blocks+1) && (file_header->total_records > totales_leidos)){ res = read_from_file(block_file, current_block, file_header, current_block->number+1); bloques_leidos++; registros_leidos = 0; current_pos = 0; }else{ todo_leido = 0; } } } /* Creo el primer sort file */ sprintf(path,"tmp/sort%d.txt",tmp_files); tmp_file = fopen(path, "w"); if(file_header->total_records == totales_leidos) heapify(buffer,totales_leidos, fp); else heapify(buffer,TMP_BUFFER, fp); while((todo_leido == 1) && tmp_file){ if(registros_leidos < current_block->total_reg){ current_pos += read_block(current_block, record, current_pos); registros_leidos++; registro_en_memoria = 0; registro_procesado = 1; }else{ if(bloques_leidos < file_header->total_blocks){ res = read_from_file(block_file, current_block, file_header, current_block->number+1); bloques_leidos++; current_pos = 0; registros_leidos = 0; registro_procesado = 1; registro_en_memoria = 1; }else{ todo_leido = 0; } } while((registro_procesado == 1) && (registro_en_memoria == 0)){ if(fp(record,&buffer[0]) < 0){ if(current_pos_freeze<TMP_BUFFER){ memcpy(&freeze_buffer[current_pos_freeze],record,sizeof(struct record_t)); current_pos_freeze++; registro_procesado = 0; registro_en_memoria = 1; } else { /* vacio el buffer en el archivo */ print_buffer_to_file(buffer, TMP_BUFFER, tmp_file, fp); /* cierro archivo y abro el siguiente */ tmp_files++; fclose(tmp_file); sprintf(path,"tmp/sort%d.txt",tmp_files); tmp_file = fopen(path, "w"); /* copio el freeze_buffer al buffer y creo el heap*/ heapify(freeze_buffer,TMP_BUFFER, fp); memcpy(buffer,freeze_buffer,sizeof(struct record_t) * TMP_BUFFER); current_pos_freeze = 0; registro_procesado = 1; } } else{ /* Guardo en el archivo, libero memoria */ print_record_to_file(buffer, tmp_file); /* Guardo en el buffer el record leido */ memcpy(&buffer[0],&buffer[TMP_BUFFER-1],sizeof(struct record_t)); memcpy(&buffer[TMP_BUFFER-1],record,sizeof(struct record_t)); registro_procesado = 0; registro_en_memoria = 1; update(buffer, TMP_BUFFER, fp); } } } /* Guardo en el archivo*/ if((file_header->total_records == totales_leidos) && (current_block->number == 1)) print_buffer_to_file(buffer, totales_leidos, tmp_file, fp); else print_buffer_to_file(buffer, TMP_BUFFER, tmp_file, fp); fclose(tmp_file); /* Guardo lo que tenia en el freeze_buffer en otro archivo */ if(current_pos_freeze > 1){ tmp_files++; sprintf(path,"tmp/sort%d.txt",tmp_files); tmp_file = fopen(path, "w"); print_buffer_to_file(freeze_buffer, current_pos_freeze, tmp_file, fp); fclose(tmp_file); } } free(record); free(current_block); free(file_header); fclose(block_file); /* Merge de los archivos generados */ res = merge_files(tmp_files, fp); printf("\n"); return res; }
// Run method for the thread // It is meant to do the following: // (1) Initialize status buffer // (2) Set up network parameters and socket // (3) Start main loop // (3a) Receive packet on socket // (3b) Error check packet (packet size, etc) // (3c) Call process_packet on received packet // (4) Terminate thread cleanly static void *run(hashpipe_thread_args_t * args) { fprintf(stdout, "N_INPUTS = %d\n", N_INPUTS); fprintf(stdout, "N_CHAN = %d\n", N_CHAN); fprintf(stdout, "N_CHAN_PER_X = %d\n", N_CHAN_PER_X); fprintf(stdout, "N_CHAN_PER_PACKET = %d\n", N_CHAN_PER_PACKET); fprintf(stdout, "N_TIME_PER_PACKET = %d\n", N_TIME_PER_PACKET); fprintf(stdout, "N_TIME_PER_BLOCK = %d\n", N_TIME_PER_BLOCK); fprintf(stdout, "N_BYTES_PER_BLOCK = %d\n", N_BYTES_PER_BLOCK); fprintf(stdout, "N_BYTES_PER_PACKET = %d\n", N_BYTES_PER_PACKET); fprintf(stdout, "N_PACKETS_PER_BLOCK = %d\n", N_PACKETS_PER_BLOCK); fprintf(stdout, "N_COR_MATRIX = %d\n", N_COR_MATRIX); // Local aliases to shorten access to args fields // Our output buffer happens to be a paper_input_databuf flag_input_databuf_t *db = (flag_input_databuf_t *)args->obuf; hashpipe_status_t st = args->st; const char * status_key = args->thread_desc->skey; st_p = &st; // allow global (this source file) access to the status buffer /* Read network params */ fprintf(stdout, "Setting up network parameters\n"); struct hashpipe_udp_params up = { .bindhost = "0.0.0.0", .bindport = 8511, .packet_size = 8008 }; hashpipe_status_lock_safe(&st); // Get info from status buffer if present (no change if not present) hgets(st.buf, "BINDHOST", 80, up.bindhost); hgeti4(st.buf, "BINDPORT", &up.bindport); // Store bind host/port info etc in status buffer hputs(st.buf, "BINDHOST", up.bindhost); hputi4(st.buf, "BINDPORT", up.bindport); hputu4(st.buf, "MISSEDFE", 0); hputu4(st.buf, "MISSEDPK", 0); hputs(st.buf, status_key, "running"); hashpipe_status_unlock_safe(&st); struct hashpipe_udp_packet p; /* Give all the threads a chance to start before opening network socket */ int netready = 0; int corready = 0; int checkready = 0; while (!netready) { sleep(1); // Check the correlator to see if it's ready yet hashpipe_status_lock_safe(&st); hgeti4(st.buf, "CORREADY", &corready); hgeti4(st.buf, "SAVEREADY", &checkready); hashpipe_status_unlock_safe(&st); if (!corready) { continue; } //if (!checkready) { // continue; //} // Check the other threads to see if they're ready yet // TBD // If we get here, then all threads are initialized netready = 1; } sleep(3); /* Set up UDP socket */ fprintf(stderr, "NET: BINDHOST = %s\n", up.bindhost); fprintf(stderr, "NET: BINDPORT = %d\n", up.bindport); int rv = hashpipe_udp_init(&up); if (rv!=HASHPIPE_OK) { hashpipe_error("paper_net_thread", "Error opening UDP socket."); pthread_exit(NULL); } pthread_cleanup_push((void *)hashpipe_udp_close, &up); // Initialize first few blocks in the buffer int i; for (i = 0; i < 2; i++) { // Wait until block semaphore is free if (flag_input_databuf_wait_free(db, i) != HASHPIPE_OK) { if (errno == EINTR) { // Interrupt occurred hashpipe_error(__FUNCTION__, "waiting for free block interrupted\n"); pthread_exit(NULL); } else { hashpipe_error(__FUNCTION__, "error waiting for free block\n"); pthread_exit(NULL); } } initialize_block(db, i*Nm); } // Set correlator to "start" state hashpipe_status_lock_safe(&st); hputs(st.buf, "INTSTAT", "start"); hashpipe_status_unlock_safe(&st); /* Main loop */ uint64_t packet_count = 0; fprintf(stdout, "Net: Starting Thread!\n"); while (run_threads()) { // Get packet do { p.packet_size = recv(up.sock, p.data, HASHPIPE_MAX_PACKET_SIZE, 0); } while (p.packet_size == -1 && (errno == EAGAIN || errno == EWOULDBLOCK) && run_threads()); if(!run_threads()) break; if (up.packet_size != p.packet_size && up.packet_size != p.packet_size-8) { // If an error was returned instead of a valid packet size if (p.packet_size == -1) { fprintf(stderr, "uh oh!\n"); // Log error and exit hashpipe_error("paper_net_thread", "hashpipe_udp_recv returned error"); perror("hashpipe_udp_recv"); pthread_exit(NULL); } else { // Log warning and ignore wrongly sized packet hashpipe_warn("paper_net_thread", "Incorrect pkt size (%d)", p.packet_size); continue; } } packet_count++; process_packet(db, &p); /* Will exit if thread has been cancelled */ pthread_testcancel(); } pthread_cleanup_pop(1); /* Closes push(hashpipe_udp_close) */ hashpipe_status_lock_busywait_safe(&st); hputs(st.buf, status_key, "terminated"); hashpipe_status_unlock_safe(&st); return NULL; } static hashpipe_thread_desc_t net_thread = { name: "flag_net_thread", skey: "NETSTAT", init: NULL, run: run, ibuf_desc: {NULL},
// This function returns -1 unless the given packet causes a block to be marked // as filled in which case this function returns the marked block's first mcnt. // Any return value other than -1 will be stored in the status memory as // NETMCNT, so it is important that values other than -1 are returned rarely // (i.e. when marking a block as filled)!!! static inline uint64_t write_paper_packet_to_blocks(paper_input_databuf_t *paper_input_databuf_p, struct guppi_udp_packet *p) { static block_info_t binfo; packet_header_t pkt_header; const uint64_t *payload_p; int rv; int i; uint64_t *dest_p; uint64_t netmcnt = -1; // Value to store in status memory #if N_DEBUG_INPUT_BLOCKS == 1 static uint64_t debug_remaining = -1ULL; static off_t debug_offset = 0; uint64_t * debug_ptr; #endif // housekeeping for each packet get_header(p, &pkt_header); #if N_DEBUG_INPUT_BLOCKS == 1 debug_ptr = (uint64_t *)&paper_input_databuf_p->block[N_INPUT_BLOCKS]; debug_ptr[debug_offset++] = be64toh(*(uint64_t *)(p->data)); if(--debug_remaining == 0) { exit(1); } if(debug_offset >= sizeof(paper_input_block_t)/sizeof(uint64_t)) { debug_offset = 0; } #endif if(! binfo.initialized) { // insist that we start on a multiple of sub_blocks/block if(pkt_header.mcnt % N_SUB_BLOCKS_PER_INPUT_BLOCK != 0) { return -1; } initialize_block_info(paper_input_databuf_p, &binfo, pkt_header.mcnt); } if(out_of_seq_mcnt(&binfo, pkt_header.mcnt)) { return(handle_out_of_seq_mcnt(&binfo)); } if((rv = calc_block_indexes(&binfo, &pkt_header))) { return rv; } if(! binfo.block_active[binfo.block_i]) { // new block, pass along the block for N_INPUT_BLOCKS/2 blocks ago i = subtract_block_i(binfo.block_i, N_INPUT_BLOCKS/2); #if N_DEBUG_INPUT_BLOCKS == 1 #define DEBUG_EXTRA (0x10000) if(set_block_filled(paper_input_databuf_p, &binfo, i) && debug_remaining > DEBUG_EXTRA) { debug_remaining = DEBUG_EXTRA; } #else set_block_filled(paper_input_databuf_p, &binfo, i); #endif netmcnt = paper_input_databuf_p->block[i].header.mcnt; // Wait (hopefully not long!) for free block for this packet if((rv = paper_input_databuf_busywait_free(paper_input_databuf_p, binfo.block_i)) != GUPPI_OK) { if (rv==GUPPI_TIMEOUT) { // run_threads is 0 (i.e. shutting down) return -1; } else { guppi_error(__FUNCTION__, "error waiting for free databuf"); run_threads=0; pthread_exit(NULL); return -1; } } initialize_block(paper_input_databuf_p, &binfo, pkt_header.mcnt); } binfo.block_active[binfo.block_i]++; // increment packet count for block // end housekeeping // Calculate starting points for unpacking this packet into block's data buffer. dest_p = paper_input_databuf_p->block[binfo.block_i].data + paper_input_databuf_data_idx(binfo.m, binfo.x, binfo.q, binfo.f, 0, 0); payload_p = (uint64_t *)(p->data+8); // Copy data into buffer memcpy(dest_p, payload_p, N_BYTES_PER_PACKET); return netmcnt; }
int main(int argc, char **argv){ int i=0, res, free_space; char content[17] = "Esto es un test "; FILE * block_file; struct file_header * fh; struct block_t * block; struct record_t * record; printf("Welcome to Carga Masiva Test\n\n"); block_file = fopen("block_records_test.dat", "w"); fh = (struct file_header*) malloc(sizeof(struct file_header)); record = (struct record_t*) malloc(sizeof(struct record_t)); block = (struct block_t *) malloc(sizeof(struct block_t)); if(!record || !block || !fh) return -1; initialize_block(block, 1); initialize_record(record); initialize_file_header(fh); res = write_header(block_file, fh); if(res == 0){ for(i=0;i<50;i++){ record->content = (char*) malloc(strlen(content)+1); memcpy(record->content, content, strlen(content)+1); record->content[17] = '\0'; record->code = i+1; record->size = (int)sizeof(record->size) + (int)sizeof(record->code) + (int) strlen(record->content) + 1; free_space = MAX_SIZE - block->space_occupied; if(free_space < record->size){ write_to_file(block_file, fh, block); printf("(Bloque %d) \tOcupado: %d, Records: %d\n", block->number, block->space_occupied, block->total_reg); initialize_block(block, block->number+1); } write_block(block, fh, record); initialize_record(record); free(record->content); } } write_to_file(block_file, fh, block); printf("(Bloque %d) \tOcupado: %d, Records: %d\n", block->number, block->space_occupied, block->total_reg); printf("\nListo la carga de bloques\n"); printf("---------------------------------------\n"); print_header(fh); free(block); free(record); free(fh); fclose(block_file); return 0; }