void sstdata_open(sst_data_t* sstdata) { int ret; int id; int i; int filterlen = 0; struct _stat info; _stat(sstdata->filename, &info); if(info.st_size <12) { sstdata->key_num = 0; __INFO("file content error:%s",sstdata->filename); } else { sstdata->file = fopen(sstdata->filename,"rb"); ret = fread(buffer_detach(sstdata->buf),12,1,sstdata->file); sstdata->buf->NUL = 16; id = buffer_getint(sstdata->buf); sstdata->key_num = buffer_getint(sstdata->buf); sstdata->max = buffer_getint(sstdata->buf); if(sstdata->id != id) { __INFO("file content id:%d not equal read id:%d",sstdata->id,id); } if(info.st_size - 12 > sstdata->buf->buflen) { buffer_free(sstdata->buf); sstdata->buf = buffer_new(info.st_size - 12); } buffer_clear(sstdata->buf); buffer_seekfirst(sstdata->buf); fseek(sstdata->file,12+filterlen,SEEK_SET); ret = fread(buffer_detach(sstdata->buf),1,info.st_size - 12 - filterlen,sstdata->file); i = sstdata->key_num; sstdata->keys = (data_t*)xmalloc(sstdata->key_num * sizeof(data_t*)); for (i=0; i<sstdata->key_num; i++) { sstdata->keys[i] = buffer_getdata(sstdata->buf); } sstdata->bigest_key = sstdata->keys[sstdata->key_num-1]; sstdata->smallest_key = sstdata->keys[0]; } }
//write sstable data in file void sstdata_writedata(sst_data_t* sstdata) { int ret,i; for (i=0; i<sstdata->key_num; i++) { buffer_putdata(sstdata->buf,sstdata->keys[i]); if (sstdata->buf->NUL > 8192) { ret = fwrite(buffer_detach(sstdata->buf),sstdata->buf->NUL,1,sstdata->file); buffer_clear(sstdata->buf); } } ret = fwrite(buffer_detach(sstdata->buf),sstdata->buf->NUL,1,sstdata->file); fflush(sstdata->file); }
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Parse Falcon's webpage containing the list of 1-minute * resolution CSV files and populate file_list with the * names of these files. */ int csv_get_file_list( csv_context_t *file_list, buffer_t* buf ) { int result = 1; int i = 0; regex_t regex; regmatch_t match_list[MAX_MATCHES]; regmatch_t* match = NULL; buffer_t* file_name = NULL; char* content_string = NULL; size_t max_off = 0; if (buffer_size(buf)) { file_name = buffer_init(); memset(&match_list, 0, sizeof(match_list)); content_string = (char*)buf->content; if (regcomp(®ex, "HREF[=]\"(/data/[^\"]+?[.]csv)\"", REG_EXTENDED)) { goto unclean; } // Find the names of all the csv files with minute resolution while (!regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0)) { match = match_list; match++; max_off = 0; for (i = 1; i < MAX_MATCHES; i++, match++) { if (match->rm_so && (match->rm_eo > match->rm_so)) { // Add this file name to the list buffer_write(file_name, (uint8_t*)(content_string + match->rm_so), (size_t)(match->rm_eo - match->rm_so) ); buffer_terminate(file_name); if (gDebug) { printf("found CSV file: %s\n", file_name->content); } list_append(file_list, buffer_detach(file_name)); if (max_off < match->rm_eo) { max_off = match->rm_eo; } } } content_string += max_off; memset(&match_list, 0, sizeof(match_list)); } } goto clean; unclean: result = 0; clean: regfree(®ex); file_name = buffer_destroy(file_name); return result; }
int sstdata_compactput( sst_data_t* sstdata,data_t* data ) { int ret; if(sstdata->max > sstdata->key_num) { buffer_clear(sstdata->buf); buffer_putdata(sstdata->buf,data); ret = fwrite(buffer_detach(sstdata->buf),sstdata->buf->NUL,1,sstdata->file); sstdata->key_num++; return 0; } else { __INFO("file is full"); return 1; } }
//write sstable head in file void sstdata_writehead(sst_data_t* sstdata) { int ret; buffer_clear(sstdata->buf); buffer_putint(sstdata->buf,sstdata->id); buffer_putint(sstdata->buf,sstdata->key_num); buffer_putint(sstdata->buf,sstdata->max); __INFO("buffer NUL:%d\n",sstdata->buf->NUL); fseek(sstdata->file,0,SEEK_SET); ret = fwrite(buffer_detach(sstdata->buf),1,sstdata->buf->NUL,sstdata->file); fflush(sstdata->file); __INFO("sstdata:flush data head flush : %s, NUL:%d, ret:%d ",sstdata->filename,sstdata->buf->NUL,ret); buffer_clear(sstdata->buf); }
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Strip out any lines that are internal Falcon issues, append * the remaining lines to the alarm context list. */ int alarm_filter_lines( alarm_context_t* alarm_list, buffer_t* buf, time_t last_alarm_time ) { int result = 1; regex_t regex; regmatch_t match_list[MAX_MATCHES]; regmatch_t* match = NULL; regmatch_t* code_match = NULL; regmatch_t* desc_match = NULL; regmatch_t* evt_match = NULL; regmatch_t* time_match = NULL; buffer_t* line = NULL; alarm_line_t* line_element = NULL; char* content_string = NULL; char tmp_char = '\0'; size_t code_si = 0; size_t code_ei = 0; size_t desc_si = 0; size_t desc_ei = 0; size_t evt_si = 0; size_t evt_ei = 0; size_t time_si = 0; size_t time_ei = 0; struct tm time_struct; uint16_t code = 0; // Set up the csv directory url if (buffer_size(buf) && alarm_list) { // Construct the regular expression for filtering alarm messages if (regcomp(®ex, "AH([0-9]{3})[-]([0-9]{4})[-]([^ ]+)[ ]*[-]([0-9]{2}[/][0-9]{2}[/][0-9]{2} [0-9]{2}[:][0-9]{2}[:][0-9]{2})[^:]*? ([^: ]+)([:][^\n\r]+)", REG_EXTENDED | REG_NEWLINE)) { goto unclean; } memset(&match_list, 0, sizeof(match_list)); content_string = (char*)buf->content; // Only process lines we are interested in while (!regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0)) { match = match_list; code_match = &match_list[2]; evt_match = &match_list[3]; time_match = &match_list[4]; desc_match = &match_list[5]; if (match->rm_so && (match->rm_eo > match->rm_so)) { if (gDebug) printf("Parsing Alarm\n"); line_element = alarm_line_init(); if (!line_element) goto unclean; line = buffer_init(); if (!line) goto unclean; buffer_write(line, (uint8_t*)(content_string + match->rm_so), (size_t)(match->rm_eo - match->rm_so)); buffer_terminate(line); line_element->text = (char*)buffer_detach(line); line = buffer_destroy(line); line_element->hash = murmur_32(line_element->text, strlen(line_element->text), HASH_SEED_32); if (gDebug) printf(" Text: %s\n", line_element->text); // Parse time time_si = time_match->rm_so - match->rm_so; time_ei = time_match->rm_eo - match->rm_so; tmp_char = line_element->text[time_ei]; line_element->text[time_ei] = '\0'; // temporarily terminate at end of time if (gDebug) printf(" Timestamp: %s\n", line_element->text + time_si); memset(&time_struct, 0, sizeof(struct tm)); strptime((const char*)(line_element->text + time_si), "%m/%d/%y %H:%M:%S", &time_struct); line_element->timestamp = mktime(&time_struct); line_element->text[time_ei] = tmp_char; // restore character // Get the event code code_si = code_match->rm_so - match->rm_so; code_ei = code_match->rm_eo - match->rm_so; tmp_char = line_element->text[code_ei]; line_element->text[code_ei] = '\0'; if (gDebug) printf(" Code: %s\n", line_element->text + code_si); code = atoi(line_element->text + code_si); line_element->channel = code / 10; line_element->event = code % 10; line_element->text[code_ei] = tmp_char; // Determine if the alarm is a trigger or return evt_si = evt_match->rm_so - match->rm_so; evt_ei = evt_match->rm_eo - match->rm_so; tmp_char = line_element->text[evt_ei]; line_element->text[evt_ei] = '\0'; if (gDebug) printf(" Event: %s\n", line_element->text + evt_si); if (!strcmp("RTN", line_element->text + evt_si)) { line_element->event |= 0x80; } line_element->text[evt_ei] = tmp_char; // Record the description code desc_si = desc_match->rm_so - match->rm_so; desc_ei = desc_match->rm_eo - match->rm_so; tmp_char = line_element->text[desc_ei]; line_element->text[desc_ei] = '\0'; if (gDebug) printf(" Description: %s\n", line_element->text + desc_si); strncpy(line_element->description, line_element->text + desc_si, 8); line_element->description[8] = '\0'; line_element->text[desc_ei] = tmp_char; // If this is a duplicate message, throw it out if ((line_element->timestamp <= last_alarm_time) || (list_locate(alarm_list, line_element) > -1)) { free(line_element->text); free(line_element); } // Otherwise, add it to the list else { list_append(alarm_list, line_element); if ( list_size(alarm_list) > MAX_ALARMS ) { line_element = list_fetch(alarm_list); line_element = alarm_line_destroy(line_element); } } } content_string += match->rm_eo; memset(&match_list, 0, sizeof(match_list)); } } goto clean; unclean: result = 0; line_element = alarm_line_destroy(line_element); line = buffer_destroy(line); clean: regfree(®ex); return result; } // alarm_filter_lines()
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Given the contents of a CSV file, populate/update a * csv_buffer_t structure. */ int csv_parse_file( csv_buffer_t* csv_buffer, buffer_t* buf, time_t initial_time ) { int result = 1; regex_t regex; regmatch_t match_list[MAX_MATCHES]; regmatch_t* match = NULL; csv_row_t* csv_row = NULL; buffer_t* description = NULL; char* content_string = NULL; char* timestamp = NULL; char* average = NULL; char* high = NULL; char* low = NULL; struct tm time_struct; char tmp_char = '\0'; csv_header_t* csv_header; csv_context_t* csv_list; if (!csv_buffer || !buf || !buf->content) { goto unclean; } csv_header = csv_buffer->header; csv_list = csv_buffer->list; description = buffer_init(); content_string = (char *)buf->content; // Find the channel in the CSV file if (regcomp(®ex, "^Chan:,(.*)$", REG_EXTENDED | REG_NEWLINE)) { goto unclean; } if (regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0)) { goto unclean; } tmp_char = content_string[match_list[1].rm_eo]; content_string[match_list[1].rm_eo] = '\0'; csv_header->channel = atol(content_string + match_list[1].rm_so); content_string[match_list[1].rm_eo] = tmp_char; regfree(®ex); // Find the description in the CSV file if (regcomp(®ex, "^Desc:,([^:\r\n)]+)[:](.*)$", REG_EXTENDED | REG_NEWLINE)) { goto unclean; } if (regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0)) { regfree(®ex); if (regcomp(®ex, "^Desc:,([^\r\n]*)$", REG_EXTENDED | REG_NEWLINE)) { goto unclean; } if (regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0)) { goto unclean; } } buffer_write(description, (uint8_t*)(content_string + match_list[1].rm_so), match_list[1].rm_eo - match_list[1].rm_so); buffer_terminate(description); tmp_char = content_string[match_list[1].rm_eo]; content_string[match_list[1].rm_eo] = '\0'; if (csv_header->description) { free(csv_header->description); } csv_header->description = (char *)buffer_detach(description); description = buffer_destroy(description); content_string[match_list[1].rm_eo] = tmp_char; regfree(®ex); /* build regex for parsing CSV file lines */ if (regcomp(®ex, "^([0-9]{2}/[0-9]{2}/[0-9]{2},[0-9]{1,2}:[0-9]{2}),([0-9-]+),([0-9-]+),([0-9-]+)$", REG_EXTENDED | REG_NEWLINE)) { goto unclean; } // Locate and break down CSV file lines while (!regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0)) { // Null terminate to simplify match = match_list; match++; timestamp = content_string + match->rm_so; content_string[match->rm_eo] = '\0'; match++; average = content_string + match->rm_so; content_string[match->rm_eo] = '\0'; match++; high = content_string + match->rm_so; content_string[match->rm_eo] = '\0'; match++; low = content_string + match->rm_so; content_string[match->rm_eo] = '\0'; content_string += match->rm_eo + 1; csv_row = csv_row_init(); if (!csv_row) { fprintf(stderr, "Could not allocate memory for csv row.\n"); goto unclean; } // Parse the timestamp in this row memset(&time_struct, 0, sizeof(struct tm)); strptime(timestamp, "%m/%d/%y,%H:%M", &time_struct); csv_row->timestamp = mktime(&time_struct); csv_row->average = (int32_t)atol(average); csv_row->high = (int32_t)atol(high); csv_row->low = (int32_t)atol(low); csv_row->empty = 0; // Make sure we don't duplicate lines if ( (csv_buffer->end_time < csv_row->timestamp) && ((!initial_time) || (csv_row->timestamp > initial_time)) ) { if (csv_buffer->start_time == 0) { csv_buffer->start_time = csv_row->timestamp; } // Add a new row to the list list_append(csv_list, csv_row); csv_buffer->end_time = csv_row->timestamp; } else { csv_row = csv_row_destroy(csv_row); } memset(&match_list, 0, sizeof(match_list)); } goto clean; unclean: result = 0; clean: regfree(®ex); description = buffer_destroy(description); return result; }
void *_write_mmap(struct silopit *silopit, struct skipnode *x, size_t count, int need_new) { int i, j, c_clone; int fd; int sizes; int result; char file[FILE_PATH_SIZE]; struct skipnode *last; struct footer footer; struct stats stats; int fsize = sizeof(struct footer); memset(&footer, 0, fsize); _prepare_stats(x, count, &stats); sizes = stats.mmap_size; struct inner_block { char key[stats.max_len]; char offset[8]; }; struct inner_block *blks; memset(file, 0, FILE_PATH_SIZE); snprintf(file, FILE_PATH_SIZE, "%s/%s", silopit->basedir, silopit->name); fd = open(file, O_RDWR | O_CREAT | O_TRUNC, 0644); if (fd == -1) __PANIC("error creating silopit file"); if (lseek(fd, sizes - 1, SEEK_SET) == -1) __PANIC("error lseek silopit"); result = write(fd, "", 1); if (result == -1) __PANIC("error writing empty"); blks = mmap(0, sizes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if (blks == MAP_FAILED) { __PANIC("error mapping block when write on process"); } last = x; c_clone = count; for (i = 0, j = 0; i < c_clone; i++) { if (x->opt == ADD) { buffer_putstr(silopit->buf, x->key); buffer_putc(silopit->buf, 0); buffer_putlong(silopit->buf, x->val); j++; } else count--; last = x; x = x->forward[0]; } char *strings = buffer_detach(silopit->buf); memcpy(blks, strings, sizes); #ifdef MSYNC if (msync(blks, sizes, MS_SYNC) == -1) { __ERROR("Error Msync"); } #endif if (munmap(blks, sizes) == -1) { __ERROR("Un-mmapping the file"); } footer.count = to_be32(count); footer.crc = to_be32(F_CRC); footer.size = to_be32(sizes); footer.max_len = to_be32(stats.max_len); memcpy(footer.key, last->key, strlen(last->key)); result = write(fd, &footer, fsize); if (result == -1) __PANIC("writing the footer"); struct meta_node mn; mn.count = count; memset(mn.end, 0, SILOKATANA_MAX_KEY_SIZE); memcpy(mn.end, last->key, SILOKATANA_MAX_KEY_SIZE); memset(mn.index_name, 0, FILE_NAME_SIZE); memcpy(mn.index_name, silopit->name, FILE_NAME_SIZE); if (need_new) meta_set(silopit->meta, &mn); else meta_set_byname(silopit->meta, &mn); close(fd); return x; }