void sstdata_open(sst_data_t* sstdata)
{
int ret;
int id;
int i;
int filterlen = 0;
struct _stat info;
_stat(sstdata->filename, &info);
if(info.st_size <12)
{
sstdata->key_num = 0;
__INFO("file content error:%s",sstdata->filename);
}
else
{
sstdata->file = fopen(sstdata->filename,"rb");
ret = fread(buffer_detach(sstdata->buf),12,1,sstdata->file);
sstdata->buf->NUL = 16;
id = buffer_getint(sstdata->buf);
sstdata->key_num = buffer_getint(sstdata->buf);
sstdata->max = buffer_getint(sstdata->buf);
if(sstdata->id != id)
{
__INFO("file content id:%d not equal read id:%d",sstdata->id,id);
}
if(info.st_size - 12 > sstdata->buf->buflen)
{
buffer_free(sstdata->buf);
sstdata->buf = buffer_new(info.st_size - 12);
}
buffer_clear(sstdata->buf);
buffer_seekfirst(sstdata->buf);
fseek(sstdata->file,12+filterlen,SEEK_SET);
ret = fread(buffer_detach(sstdata->buf),1,info.st_size - 12 - filterlen,sstdata->file);
i = sstdata->key_num;
sstdata->keys = (data_t*)xmalloc(sstdata->key_num * sizeof(data_t*));
for (i=0; i<sstdata->key_num; i++)
{
sstdata->keys[i] = buffer_getdata(sstdata->buf);
}
sstdata->bigest_key = sstdata->keys[sstdata->key_num-1];
sstdata->smallest_key = sstdata->keys[0];
}
}
//write sstable data in file
void sstdata_writedata(sst_data_t* sstdata)
{
int ret,i;
for (i=0; i<sstdata->key_num; i++)
{
buffer_putdata(sstdata->buf,sstdata->keys[i]);
if (sstdata->buf->NUL > 8192)
{
ret = fwrite(buffer_detach(sstdata->buf),sstdata->buf->NUL,1,sstdata->file);
buffer_clear(sstdata->buf);
}
}
ret = fwrite(buffer_detach(sstdata->buf),sstdata->buf->NUL,1,sstdata->file);
fflush(sstdata->file);
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Parse Falcon's webpage containing the list of 1-minute
* resolution CSV files and populate file_list with the
* names of these files.
*/
int csv_get_file_list( csv_context_t *file_list, buffer_t* buf )
{
int result = 1;
int i = 0;
regex_t regex;
regmatch_t match_list[MAX_MATCHES];
regmatch_t* match = NULL;
buffer_t* file_name = NULL;
char* content_string = NULL;
size_t max_off = 0;
if (buffer_size(buf))
{
file_name = buffer_init();
memset(&match_list, 0, sizeof(match_list));
content_string = (char*)buf->content;
if (regcomp(®ex, "HREF[=]\"(/data/[^\"]+?[.]csv)\"", REG_EXTENDED))
{
goto unclean;
}
// Find the names of all the csv files with minute resolution
while (!regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0))
{
match = match_list;
match++;
max_off = 0;
for (i = 1; i < MAX_MATCHES; i++, match++)
{
if (match->rm_so && (match->rm_eo > match->rm_so))
{
// Add this file name to the list
buffer_write(file_name, (uint8_t*)(content_string + match->rm_so),
(size_t)(match->rm_eo - match->rm_so) );
buffer_terminate(file_name);
if (gDebug) {
printf("found CSV file: %s\n", file_name->content);
}
list_append(file_list, buffer_detach(file_name));
if (max_off < match->rm_eo)
{
max_off = match->rm_eo;
}
}
}
content_string += max_off;
memset(&match_list, 0, sizeof(match_list));
}
}
goto clean;
unclean:
result = 0;
clean:
regfree(®ex);
file_name = buffer_destroy(file_name);
return result;
}
int sstdata_compactput( sst_data_t* sstdata,data_t* data )
{
int ret;
if(sstdata->max > sstdata->key_num)
{
buffer_clear(sstdata->buf);
buffer_putdata(sstdata->buf,data);
ret = fwrite(buffer_detach(sstdata->buf),sstdata->buf->NUL,1,sstdata->file);
sstdata->key_num++;
return 0;
}
else
{
__INFO("file is full");
return 1;
}
}
//write sstable head in file
void sstdata_writehead(sst_data_t* sstdata)
{
int ret;
buffer_clear(sstdata->buf);
buffer_putint(sstdata->buf,sstdata->id);
buffer_putint(sstdata->buf,sstdata->key_num);
buffer_putint(sstdata->buf,sstdata->max);
__INFO("buffer NUL:%d\n",sstdata->buf->NUL);
fseek(sstdata->file,0,SEEK_SET);
ret = fwrite(buffer_detach(sstdata->buf),1,sstdata->buf->NUL,sstdata->file);
fflush(sstdata->file);
__INFO("sstdata:flush data head flush : %s, NUL:%d, ret:%d ",sstdata->filename,sstdata->buf->NUL,ret);
buffer_clear(sstdata->buf);
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Strip out any lines that are internal Falcon issues, append
* the remaining lines to the alarm context list.
*/
int alarm_filter_lines( alarm_context_t* alarm_list, buffer_t* buf,
time_t last_alarm_time )
{
int result = 1;
regex_t regex;
regmatch_t match_list[MAX_MATCHES];
regmatch_t* match = NULL;
regmatch_t* code_match = NULL;
regmatch_t* desc_match = NULL;
regmatch_t* evt_match = NULL;
regmatch_t* time_match = NULL;
buffer_t* line = NULL;
alarm_line_t* line_element = NULL;
char* content_string = NULL;
char tmp_char = '\0';
size_t code_si = 0;
size_t code_ei = 0;
size_t desc_si = 0;
size_t desc_ei = 0;
size_t evt_si = 0;
size_t evt_ei = 0;
size_t time_si = 0;
size_t time_ei = 0;
struct tm time_struct;
uint16_t code = 0;
// Set up the csv directory url
if (buffer_size(buf) && alarm_list)
{
// Construct the regular expression for filtering alarm messages
if (regcomp(®ex, "AH([0-9]{3})[-]([0-9]{4})[-]([^ ]+)[ ]*[-]([0-9]{2}[/][0-9]{2}[/][0-9]{2} [0-9]{2}[:][0-9]{2}[:][0-9]{2})[^:]*? ([^: ]+)([:][^\n\r]+)", REG_EXTENDED | REG_NEWLINE))
{
goto unclean;
}
memset(&match_list, 0, sizeof(match_list));
content_string = (char*)buf->content;
// Only process lines we are interested in
while (!regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0))
{
match = match_list;
code_match = &match_list[2];
evt_match = &match_list[3];
time_match = &match_list[4];
desc_match = &match_list[5];
if (match->rm_so && (match->rm_eo > match->rm_so))
{
if (gDebug) printf("Parsing Alarm\n");
line_element = alarm_line_init();
if (!line_element)
goto unclean;
line = buffer_init();
if (!line)
goto unclean;
buffer_write(line, (uint8_t*)(content_string + match->rm_so),
(size_t)(match->rm_eo - match->rm_so));
buffer_terminate(line);
line_element->text = (char*)buffer_detach(line);
line = buffer_destroy(line);
line_element->hash = murmur_32(line_element->text,
strlen(line_element->text),
HASH_SEED_32);
if (gDebug) printf(" Text: %s\n", line_element->text);
// Parse time
time_si = time_match->rm_so - match->rm_so;
time_ei = time_match->rm_eo - match->rm_so;
tmp_char = line_element->text[time_ei];
line_element->text[time_ei] = '\0'; // temporarily terminate at end of time
if (gDebug) printf(" Timestamp: %s\n", line_element->text + time_si);
memset(&time_struct, 0, sizeof(struct tm));
strptime((const char*)(line_element->text + time_si),
"%m/%d/%y %H:%M:%S", &time_struct);
line_element->timestamp = mktime(&time_struct);
line_element->text[time_ei] = tmp_char; // restore character
// Get the event code
code_si = code_match->rm_so - match->rm_so;
code_ei = code_match->rm_eo - match->rm_so;
tmp_char = line_element->text[code_ei];
line_element->text[code_ei] = '\0';
if (gDebug) printf(" Code: %s\n", line_element->text + code_si);
code = atoi(line_element->text + code_si);
line_element->channel = code / 10;
line_element->event = code % 10;
line_element->text[code_ei] = tmp_char;
// Determine if the alarm is a trigger or return
evt_si = evt_match->rm_so - match->rm_so;
evt_ei = evt_match->rm_eo - match->rm_so;
tmp_char = line_element->text[evt_ei];
line_element->text[evt_ei] = '\0';
if (gDebug) printf(" Event: %s\n", line_element->text + evt_si);
if (!strcmp("RTN", line_element->text + evt_si)) {
line_element->event |= 0x80;
}
line_element->text[evt_ei] = tmp_char;
// Record the description code
desc_si = desc_match->rm_so - match->rm_so;
desc_ei = desc_match->rm_eo - match->rm_so;
tmp_char = line_element->text[desc_ei];
line_element->text[desc_ei] = '\0';
if (gDebug) printf(" Description: %s\n", line_element->text + desc_si);
strncpy(line_element->description, line_element->text + desc_si, 8);
line_element->description[8] = '\0';
line_element->text[desc_ei] = tmp_char;
// If this is a duplicate message, throw it out
if ((line_element->timestamp <= last_alarm_time) ||
(list_locate(alarm_list, line_element) > -1))
{
free(line_element->text);
free(line_element);
}
// Otherwise, add it to the list
else
{
list_append(alarm_list, line_element);
if ( list_size(alarm_list) > MAX_ALARMS )
{
line_element = list_fetch(alarm_list);
line_element = alarm_line_destroy(line_element);
}
}
}
content_string += match->rm_eo;
memset(&match_list, 0, sizeof(match_list));
}
}
goto clean;
unclean:
result = 0;
line_element = alarm_line_destroy(line_element);
line = buffer_destroy(line);
clean:
regfree(®ex);
return result;
} // alarm_filter_lines()
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Given the contents of a CSV file, populate/update a
* csv_buffer_t structure.
*/
int csv_parse_file( csv_buffer_t* csv_buffer, buffer_t* buf, time_t initial_time )
{
int result = 1;
regex_t regex;
regmatch_t match_list[MAX_MATCHES];
regmatch_t* match = NULL;
csv_row_t* csv_row = NULL;
buffer_t* description = NULL;
char* content_string = NULL;
char* timestamp = NULL;
char* average = NULL;
char* high = NULL;
char* low = NULL;
struct tm time_struct;
char tmp_char = '\0';
csv_header_t* csv_header;
csv_context_t* csv_list;
if (!csv_buffer || !buf || !buf->content) {
goto unclean;
}
csv_header = csv_buffer->header;
csv_list = csv_buffer->list;
description = buffer_init();
content_string = (char *)buf->content;
// Find the channel in the CSV file
if (regcomp(®ex, "^Chan:,(.*)$", REG_EXTENDED | REG_NEWLINE)) {
goto unclean;
}
if (regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0)) {
goto unclean;
}
tmp_char = content_string[match_list[1].rm_eo];
content_string[match_list[1].rm_eo] = '\0';
csv_header->channel = atol(content_string + match_list[1].rm_so);
content_string[match_list[1].rm_eo] = tmp_char;
regfree(®ex);
// Find the description in the CSV file
if (regcomp(®ex, "^Desc:,([^:\r\n)]+)[:](.*)$", REG_EXTENDED | REG_NEWLINE)) {
goto unclean;
}
if (regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0)) {
regfree(®ex);
if (regcomp(®ex, "^Desc:,([^\r\n]*)$", REG_EXTENDED | REG_NEWLINE)) {
goto unclean;
}
if (regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0)) {
goto unclean;
}
}
buffer_write(description, (uint8_t*)(content_string + match_list[1].rm_so),
match_list[1].rm_eo - match_list[1].rm_so);
buffer_terminate(description);
tmp_char = content_string[match_list[1].rm_eo];
content_string[match_list[1].rm_eo] = '\0';
if (csv_header->description)
{
free(csv_header->description);
}
csv_header->description = (char *)buffer_detach(description);
description = buffer_destroy(description);
content_string[match_list[1].rm_eo] = tmp_char;
regfree(®ex);
/* build regex for parsing CSV file lines */
if (regcomp(®ex, "^([0-9]{2}/[0-9]{2}/[0-9]{2},[0-9]{1,2}:[0-9]{2}),([0-9-]+),([0-9-]+),([0-9-]+)$", REG_EXTENDED | REG_NEWLINE))
{
goto unclean;
}
// Locate and break down CSV file lines
while (!regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0))
{
// Null terminate to simplify
match = match_list; match++;
timestamp = content_string + match->rm_so;
content_string[match->rm_eo] = '\0'; match++;
average = content_string + match->rm_so;
content_string[match->rm_eo] = '\0'; match++;
high = content_string + match->rm_so;
content_string[match->rm_eo] = '\0'; match++;
low = content_string + match->rm_so;
content_string[match->rm_eo] = '\0';
content_string += match->rm_eo + 1;
csv_row = csv_row_init();
if (!csv_row)
{
fprintf(stderr, "Could not allocate memory for csv row.\n");
goto unclean;
}
// Parse the timestamp in this row
memset(&time_struct, 0, sizeof(struct tm));
strptime(timestamp, "%m/%d/%y,%H:%M", &time_struct);
csv_row->timestamp = mktime(&time_struct);
csv_row->average = (int32_t)atol(average);
csv_row->high = (int32_t)atol(high);
csv_row->low = (int32_t)atol(low);
csv_row->empty = 0;
// Make sure we don't duplicate lines
if ( (csv_buffer->end_time < csv_row->timestamp) &&
((!initial_time) || (csv_row->timestamp > initial_time)) )
{
if (csv_buffer->start_time == 0)
{
csv_buffer->start_time = csv_row->timestamp;
}
// Add a new row to the list
list_append(csv_list, csv_row);
csv_buffer->end_time = csv_row->timestamp;
}
else
{
csv_row = csv_row_destroy(csv_row);
}
memset(&match_list, 0, sizeof(match_list));
}
goto clean;
unclean:
result = 0;
clean:
regfree(®ex);
description = buffer_destroy(description);
return result;
}
void *_write_mmap(struct silopit *silopit, struct skipnode *x, size_t count, int need_new)
{
int i, j, c_clone;
int fd;
int sizes;
int result;
char file[FILE_PATH_SIZE];
struct skipnode *last;
struct footer footer;
struct stats stats;
int fsize = sizeof(struct footer);
memset(&footer, 0, fsize);
_prepare_stats(x, count, &stats);
sizes = stats.mmap_size;
struct inner_block {
char key[stats.max_len];
char offset[8];
};
struct inner_block *blks;
memset(file, 0, FILE_PATH_SIZE);
snprintf(file, FILE_PATH_SIZE, "%s/%s", silopit->basedir, silopit->name);
fd = open(file, O_RDWR | O_CREAT | O_TRUNC, 0644);
if (fd == -1)
__PANIC("error creating silopit file");
if (lseek(fd, sizes - 1, SEEK_SET) == -1)
__PANIC("error lseek silopit");
result = write(fd, "", 1);
if (result == -1)
__PANIC("error writing empty");
blks = mmap(0, sizes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
if (blks == MAP_FAILED) {
__PANIC("error mapping block when write on process");
}
last = x;
c_clone = count;
for (i = 0, j = 0; i < c_clone; i++) {
if (x->opt == ADD) {
buffer_putstr(silopit->buf, x->key);
buffer_putc(silopit->buf, 0);
buffer_putlong(silopit->buf, x->val);
j++;
} else
count--;
last = x;
x = x->forward[0];
}
char *strings = buffer_detach(silopit->buf);
memcpy(blks, strings, sizes);
#ifdef MSYNC
if (msync(blks, sizes, MS_SYNC) == -1) {
__ERROR("Error Msync");
}
#endif
if (munmap(blks, sizes) == -1) {
__ERROR("Un-mmapping the file");
}
footer.count = to_be32(count);
footer.crc = to_be32(F_CRC);
footer.size = to_be32(sizes);
footer.max_len = to_be32(stats.max_len);
memcpy(footer.key, last->key, strlen(last->key));
result = write(fd, &footer, fsize);
if (result == -1)
__PANIC("writing the footer");
struct meta_node mn;
mn.count = count;
memset(mn.end, 0, SILOKATANA_MAX_KEY_SIZE);
memcpy(mn.end, last->key, SILOKATANA_MAX_KEY_SIZE);
memset(mn.index_name, 0, FILE_NAME_SIZE);
memcpy(mn.index_name, silopit->name, FILE_NAME_SIZE);
if (need_new)
meta_set(silopit->meta, &mn);
else
meta_set_byname(silopit->meta, &mn);
close(fd);
return x;
}
