/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Update the alarm list with the latest content from the
* Falcon. Make sure there are no duplicates.
*/
void alarm_poll( alarm_context_t* alarm_list, buffer_t* url_str,
st_info_t* st_info, time_t last_alarm_time )
{
buffer_t* url = NULL;
buffer_t* buf = NULL;
const char* file_path = "/data/alarmhistory.txt";
url = buffer_init();
buffer_write(url, url_str->content, url_str->length);
buffer_write(url, (uint8_t*)file_path, strlen(file_path));
buffer_terminate(url);
// Get alarm file text
buf = buffer_init();
get_page((char*)url->content, buf);
// Getting alarm lines
alarm_filter_lines( alarm_list, buf, last_alarm_time );
// Re-order alarms from oldest to youngest
list_sort(alarm_list, 1);
// Cleanup
url = buffer_destroy(url);
buf = buffer_destroy(buf);
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Parse Falcon's webpage containing the list of 1-minute
* resolution CSV files and populate file_list with the
* names of these files.
*/
int csv_get_file_list( csv_context_t *file_list, buffer_t* buf )
{
int result = 1;
int i = 0;
regex_t regex;
regmatch_t match_list[MAX_MATCHES];
regmatch_t* match = NULL;
buffer_t* file_name = NULL;
char* content_string = NULL;
size_t max_off = 0;
if (buffer_size(buf))
{
file_name = buffer_init();
memset(&match_list, 0, sizeof(match_list));
content_string = (char*)buf->content;
if (regcomp(®ex, "HREF[=]\"(/data/[^\"]+?[.]csv)\"", REG_EXTENDED))
{
goto unclean;
}
// Find the names of all the csv files with minute resolution
while (!regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0))
{
match = match_list;
match++;
max_off = 0;
for (i = 1; i < MAX_MATCHES; i++, match++)
{
if (match->rm_so && (match->rm_eo > match->rm_so))
{
// Add this file name to the list
buffer_write(file_name, (uint8_t*)(content_string + match->rm_so),
(size_t)(match->rm_eo - match->rm_so) );
buffer_terminate(file_name);
if (gDebug) {
printf("found CSV file: %s\n", file_name->content);
}
list_append(file_list, buffer_detach(file_name));
if (max_off < match->rm_eo)
{
max_off = match->rm_eo;
}
}
}
content_string += max_off;
memset(&match_list, 0, sizeof(match_list));
}
}
goto clean;
unclean:
result = 0;
clean:
regfree(®ex);
file_name = buffer_destroy(file_name);
return result;
}
static Buffer* cookie_put_value(Buffer* cookie,
const char* name, int nlen,
const char* value, int vlen,
int boolean, int encode)
{
Buffer dnam;
Buffer dval;
buffer_wrap(&dnam, name , nlen);
buffer_wrap(&dval, value, vlen);
/* output each part into the cookie */
do {
if (cookie->pos > 0) {
buffer_append(cookie, "; ", 2);
}
if (!encode) {
buffer_append(cookie, dnam.data, dnam.size);
} else {
url_encode(&dnam, dnam.size, cookie);
}
if (!boolean) {
buffer_append(cookie, "=", 1);
if (!encode) {
buffer_append(cookie, dval.data, dval.size);
} else {
url_encode(&dval, dval.size, cookie);
}
}
} while (0);
buffer_terminate(cookie);
return cookie;
}
/*
* Given a buffer that holds a cookie (and therefore has an idea
* of the current position within the cookie), parse the next
* name / value pair out of it.
*
* A cookie will have the form:
*
* name1 = value1; name2=value2;name3 =value3;...
*
* As the example shows, there may be annoying whitespace embedded
* within the name=value components. What we do here is to run a
* state machine that keeps track of the following states:
*
* URI_STATE_START Start parsing
* URI_STATE_NAME Parsing name component
* URI_STATE_EQUALS Just saw the '=' between name and value
* URI_STATE_VALUE Parsing the value component
* URI_STATE_END End parsing
* URI_STATE_ERROR Error while parsing
*
* In order to achieve the maximum performance, this state machine
* is represented in a precomputed table called uri_state_tbl[c][s],
* whose values depend on the current character and current state.
* This table (as well as the other tables that ease the process
* of URL encoding and decoding) was generated with a C program,
* which can be found in tools/encode/encode.
*/
Buffer* cookie_get_pair(Buffer* cookie,
Buffer* name, Buffer* value)
{
int ncur = name->pos;
int vcur = value->pos;
int vend = 0;
int state = 0;
int current = 0;
/* State machine starts in URI_STATE_START state and
* will loop until we enter any state that is
* >= URI_STATE_TERMINATE */
for (state = URI_STATE_START; state < URI_STATE_TERMINATE; ) {
/* Switch to next state based on last character read
* and current state. */
current = cookie->data[cookie->pos];
state = uri_state_tbl[current][state];
switch (state) {
/* If we are reading the name part, add the current
* character (possibly URL-decoded) */
case URI_STATE_NAME:
buffer_ensure_unused(name, 1);
if (current == '%' &&
isxdigit(cookie->data[cookie->pos+1]) &&
isxdigit(cookie->data[cookie->pos+2])) {
/* put a byte together from the next two hex digits */
name->data[name->pos++] = MAKE_BYTE(uri_decode_tbl[(int)cookie->data[cookie->pos+1]],
uri_decode_tbl[(int)cookie->data[cookie->pos+2]]);
cookie->pos += 3;
} else {
/* just copy current character */
name->data[name->pos++] = current;
++cookie->pos;
}
break;
/* If we are reading the value part, add the current
* character (possibly URL-decoded) */
case URI_STATE_VALUE:
buffer_ensure_unused(value, 1);
if (current == '%' &&
isxdigit(cookie->data[cookie->pos+1]) &&
isxdigit(cookie->data[cookie->pos+2])) {
/* put a byte together from the next two hex digits */
value->data[value->pos++] = MAKE_BYTE(uri_decode_tbl[(int)cookie->data[cookie->pos+1]],
uri_decode_tbl[(int)cookie->data[cookie->pos+2]]);
cookie->pos += 3;
vend = value->pos;
} else {
/* just copy current character */
value->data[value->pos++] = current;
++cookie->pos;
if (!isspace(current)) {
vend = value->pos;
}
}
break;
/* Any other state, just move to the next position. */
default:
++cookie->pos;
break;
}
}
/* If last character seen was EOS, we have already incremented
* the buffer position once too many; correct that. */
if (current == '\0') {
--cookie->pos;
}
/* If we didn't end in URI_STATE_END, reset buffers. */
if (state != URI_STATE_END) {
name->pos = ncur;
value->pos = vcur;
} else {
/* Maybe correct end position for value. */
if (vend) {
value->pos = vend;
}
}
/* Terminate both output buffers and return. */
buffer_terminate(name);
buffer_terminate(value);
return cookie;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Write all new alarms to the diskloop.
*/
void alarm_archive( alarm_context_t* alarm_list, buffer_t* url_str,
st_info_t* st_info )
{
time_t start_time = 0;
time_t end_time = 0;
uint8_t buf_byte = 0;
uint16_t buf_word = 0;
uint32_t buf_dword = 0L;
//uint64_t buf_qword = 0LL;
uint16_t version_type = 0x8000 | FALCON_VERSION;
uint16_t alarm_count = 0;
char* retmsg = NULL;
buffer_t* alarm_data = NULL;
alarm_line_t* alarm = NULL;
alarm_data = buffer_init();
if (!alarm_data) {
if (gDebug)
fprintf(stderr, "falcon: unable to allocate space for alarm data\n");
else
syslog(LOG_ERR, "falcon: unable to allocate space for alarm data\n");
return;
}
// Print each line in the filtered list
list_iterator_stop(alarm_list);
list_iterator_start(alarm_list);
while (list_iterator_hasnext(alarm_list))
{
alarm = (alarm_line_t*)list_iterator_next(alarm_list);
if (alarm->sent)
continue;
if (!start_time)
start_time = alarm->timestamp;
if (!end_time)
end_time = alarm->timestamp;
if (gDebug)
fprintf(stdout, "DEBUG %s, line %d, date %s: %s\n",
__FILE__, __LINE__, __DATE__, alarm->text);
if ((retmsg = q330LogMsg(alarm->text, st_info->station,
st_info->network, "LOG",
st_info->location)) != NULL)
{ // error trying to log the message
if (gDebug)
fprintf(stderr, "falcon: failed to write alarms to log: %s\n", retmsg);
else
syslog(LOG_ERR, "falcon: failed to write alarms to log: %s\n", retmsg);
exit(1);
}
if (gDebug)
{
fprintf(stdout, "Alarm '%s':\n", alarm->description);
fprintf(stdout, " Channel : %02x\n", (alarm->channel));
fprintf(stdout, " Timestamp : %li\n", (long)(alarm->timestamp));
fprintf(stdout, " Event Code : 0x%02x\n", (alarm->event));
fprintf(stdout, " Sent : %s\n", alarm->sent ? "Yes" : "No");
fprintf(stdout, " Hash : 0x%08lx\n", (unsigned long)(alarm->hash));
fprintf(stdout, " Text : %s\n", alarm->text);
}
if (start_time > alarm->timestamp)
start_time = alarm->timestamp;
if (end_time < alarm->timestamp)
end_time = alarm->timestamp;
if (!alarm->description[0])
{
if (gDebug)
fprintf(stderr, "falcon: description code not found\n");
else
syslog(LOG_ERR, "falcon: description code not found\n");
continue;
}
// There must be at least enough space for one more alarm.
// If there isn't, queue this buffer's contents, and reset it.
// This should prevent us from ever fragmenting alarm data
// across opaque blockettes.
if (alarm_count && (alarm_data->length > 400))
{
buffer_seek(alarm_data, 2);
buf_dword = htonl(start_time);
buffer_write(alarm_data, (uint8_t*)(&buf_dword), sizeof(buf_dword));
buf_dword = htonl(end_time);
buffer_write(alarm_data, (uint8_t*)(&buf_dword), sizeof(buf_dword));
buf_word = htons(alarm_count);
buffer_write(alarm_data, (uint8_t*)(&buf_word), sizeof(buf_word));
if (gDebug) {
fprintf(stdout, "falcon: [MID] alarms were found\n");
fprintf(stdout, "[MID] raw buffer:\n");
format_data(alarm_data->content, alarm_data->length, 0, 0);
}
QueueOpaque(alarm_data->content, (int)alarm_data->length,
st_info->station, st_info->network,
st_info->alarm_chan, st_info->location,
FALCON_IDSTRING);
alarm_data = buffer_destroy(alarm_data);
alarm_count = 0;
start_time = alarm->timestamp;
end_time = alarm->timestamp;
alarm_data = buffer_init();
if (!alarm_data) {
if (gDebug)
fprintf(stderr, "falcon: unable to allocate space for alarm data\n");
else
syslog(LOG_ERR, "falcon: unable to allocate space for alarm data\n");
return;
}
} // If we've built up a records worth of alarm data
if (!alarm_count)
{
// Write alarm header info
if (gDebug)
{
fprintf(stderr, "falcon: Writing alarm header info.\n");
}
buf_word = htons(version_type);
buffer_write(alarm_data, (uint8_t*)(&buf_word), sizeof(buf_word));
// Reserve space for elements that will be assigned just
// prior to queueing data
buf_dword = htonl(start_time);
buffer_write(alarm_data, (uint8_t*)(&buf_dword), sizeof(buf_dword));
buf_dword = htonl(end_time);
buffer_write(alarm_data, (uint8_t*)(&buf_dword), sizeof(buf_dword));
buf_word = htons(alarm_count);
buffer_write(alarm_data, (uint8_t*)(&buf_word), sizeof(buf_word));
}
// Add an alarm
buf_word = htons(alarm->channel);
buffer_write(alarm_data, (uint8_t*)(&buf_word), sizeof(buf_word));
buf_dword = htonl(alarm->timestamp);
buffer_write(alarm_data, (uint8_t*)(&buf_dword), sizeof(buf_dword));
buffer_write(alarm_data, &(alarm->event), sizeof(alarm->event));
buf_byte = (uint8_t)strlen(alarm->description);
buffer_write(alarm_data, &buf_byte, sizeof(buf_byte));
buffer_write(alarm_data, (uint8_t*)(alarm->description), (size_t)buf_byte);
buffer_terminate(alarm_data);
alarm_count++;
alarm->sent = 1;
}
list_iterator_stop(alarm_list);
if (alarm_count && alarm_data && alarm_data->content && alarm_data->length)
{
buffer_seek(alarm_data, 2);
buf_dword = htonl(start_time);
buffer_write(alarm_data, (uint8_t*)(&buf_dword), sizeof(buf_dword));
buf_dword = htonl(end_time);
buffer_write(alarm_data, (uint8_t*)(&buf_dword), sizeof(buf_dword));
buf_word = htons(alarm_count);
buffer_write(alarm_data, (uint8_t*)(&buf_word), sizeof(buf_word));
if (gDebug) {
fprintf(stderr, "falcon: [END] alarms were found\n");
fprintf(stderr, "[END] raw buffer:\n");
format_data(alarm_data->content, alarm_data->length, 0, 0);
}
QueueOpaque(alarm_data->content, (int)alarm_data->length,
st_info->station, st_info->network,
st_info->alarm_chan, st_info->location,
FALCON_IDSTRING);
}
alarm_data = buffer_destroy(alarm_data);
// Make sure we limit the accumulation of alarm messages
while (list_size(alarm_list) > MAX_CONTEXT_ALARMS)
{
alarm = list_fetch(alarm_list);
alarm = alarm_line_destroy(alarm);
}
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Strip out any lines that are internal Falcon issues, append
* the remaining lines to the alarm context list.
*/
int alarm_filter_lines( alarm_context_t* alarm_list, buffer_t* buf,
time_t last_alarm_time )
{
int result = 1;
regex_t regex;
regmatch_t match_list[MAX_MATCHES];
regmatch_t* match = NULL;
regmatch_t* code_match = NULL;
regmatch_t* desc_match = NULL;
regmatch_t* evt_match = NULL;
regmatch_t* time_match = NULL;
buffer_t* line = NULL;
alarm_line_t* line_element = NULL;
char* content_string = NULL;
char tmp_char = '\0';
size_t code_si = 0;
size_t code_ei = 0;
size_t desc_si = 0;
size_t desc_ei = 0;
size_t evt_si = 0;
size_t evt_ei = 0;
size_t time_si = 0;
size_t time_ei = 0;
struct tm time_struct;
uint16_t code = 0;
// Set up the csv directory url
if (buffer_size(buf) && alarm_list)
{
// Construct the regular expression for filtering alarm messages
if (regcomp(®ex, "AH([0-9]{3})[-]([0-9]{4})[-]([^ ]+)[ ]*[-]([0-9]{2}[/][0-9]{2}[/][0-9]{2} [0-9]{2}[:][0-9]{2}[:][0-9]{2})[^:]*? ([^: ]+)([:][^\n\r]+)", REG_EXTENDED | REG_NEWLINE))
{
goto unclean;
}
memset(&match_list, 0, sizeof(match_list));
content_string = (char*)buf->content;
// Only process lines we are interested in
while (!regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0))
{
match = match_list;
code_match = &match_list[2];
evt_match = &match_list[3];
time_match = &match_list[4];
desc_match = &match_list[5];
if (match->rm_so && (match->rm_eo > match->rm_so))
{
if (gDebug) printf("Parsing Alarm\n");
line_element = alarm_line_init();
if (!line_element)
goto unclean;
line = buffer_init();
if (!line)
goto unclean;
buffer_write(line, (uint8_t*)(content_string + match->rm_so),
(size_t)(match->rm_eo - match->rm_so));
buffer_terminate(line);
line_element->text = (char*)buffer_detach(line);
line = buffer_destroy(line);
line_element->hash = murmur_32(line_element->text,
strlen(line_element->text),
HASH_SEED_32);
if (gDebug) printf(" Text: %s\n", line_element->text);
// Parse time
time_si = time_match->rm_so - match->rm_so;
time_ei = time_match->rm_eo - match->rm_so;
tmp_char = line_element->text[time_ei];
line_element->text[time_ei] = '\0'; // temporarily terminate at end of time
if (gDebug) printf(" Timestamp: %s\n", line_element->text + time_si);
memset(&time_struct, 0, sizeof(struct tm));
strptime((const char*)(line_element->text + time_si),
"%m/%d/%y %H:%M:%S", &time_struct);
line_element->timestamp = mktime(&time_struct);
line_element->text[time_ei] = tmp_char; // restore character
// Get the event code
code_si = code_match->rm_so - match->rm_so;
code_ei = code_match->rm_eo - match->rm_so;
tmp_char = line_element->text[code_ei];
line_element->text[code_ei] = '\0';
if (gDebug) printf(" Code: %s\n", line_element->text + code_si);
code = atoi(line_element->text + code_si);
line_element->channel = code / 10;
line_element->event = code % 10;
line_element->text[code_ei] = tmp_char;
// Determine if the alarm is a trigger or return
evt_si = evt_match->rm_so - match->rm_so;
evt_ei = evt_match->rm_eo - match->rm_so;
tmp_char = line_element->text[evt_ei];
line_element->text[evt_ei] = '\0';
if (gDebug) printf(" Event: %s\n", line_element->text + evt_si);
if (!strcmp("RTN", line_element->text + evt_si)) {
line_element->event |= 0x80;
}
line_element->text[evt_ei] = tmp_char;
// Record the description code
desc_si = desc_match->rm_so - match->rm_so;
desc_ei = desc_match->rm_eo - match->rm_so;
tmp_char = line_element->text[desc_ei];
line_element->text[desc_ei] = '\0';
if (gDebug) printf(" Description: %s\n", line_element->text + desc_si);
strncpy(line_element->description, line_element->text + desc_si, 8);
line_element->description[8] = '\0';
line_element->text[desc_ei] = tmp_char;
// If this is a duplicate message, throw it out
if ((line_element->timestamp <= last_alarm_time) ||
(list_locate(alarm_list, line_element) > -1))
{
free(line_element->text);
free(line_element);
}
// Otherwise, add it to the list
else
{
list_append(alarm_list, line_element);
if ( list_size(alarm_list) > MAX_ALARMS )
{
line_element = list_fetch(alarm_list);
line_element = alarm_line_destroy(line_element);
}
}
}
content_string += match->rm_eo;
memset(&match_list, 0, sizeof(match_list));
}
}
goto clean;
unclean:
result = 0;
line_element = alarm_line_destroy(line_element);
line = buffer_destroy(line);
clean:
regfree(®ex);
return result;
} // alarm_filter_lines()
const char *buffer_content0(Buffer *buf) {
if (buf->len == 0 || !buffer_terminate(buf))
return "";
return buf->data;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Given the contents of a CSV file, populate/update a
* csv_buffer_t structure.
*/
int csv_parse_file( csv_buffer_t* csv_buffer, buffer_t* buf, time_t initial_time )
{
int result = 1;
regex_t regex;
regmatch_t match_list[MAX_MATCHES];
regmatch_t* match = NULL;
csv_row_t* csv_row = NULL;
buffer_t* description = NULL;
char* content_string = NULL;
char* timestamp = NULL;
char* average = NULL;
char* high = NULL;
char* low = NULL;
struct tm time_struct;
char tmp_char = '\0';
csv_header_t* csv_header;
csv_context_t* csv_list;
if (!csv_buffer || !buf || !buf->content) {
goto unclean;
}
csv_header = csv_buffer->header;
csv_list = csv_buffer->list;
description = buffer_init();
content_string = (char *)buf->content;
// Find the channel in the CSV file
if (regcomp(®ex, "^Chan:,(.*)$", REG_EXTENDED | REG_NEWLINE)) {
goto unclean;
}
if (regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0)) {
goto unclean;
}
tmp_char = content_string[match_list[1].rm_eo];
content_string[match_list[1].rm_eo] = '\0';
csv_header->channel = atol(content_string + match_list[1].rm_so);
content_string[match_list[1].rm_eo] = tmp_char;
regfree(®ex);
// Find the description in the CSV file
if (regcomp(®ex, "^Desc:,([^:\r\n)]+)[:](.*)$", REG_EXTENDED | REG_NEWLINE)) {
goto unclean;
}
if (regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0)) {
regfree(®ex);
if (regcomp(®ex, "^Desc:,([^\r\n]*)$", REG_EXTENDED | REG_NEWLINE)) {
goto unclean;
}
if (regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0)) {
goto unclean;
}
}
buffer_write(description, (uint8_t*)(content_string + match_list[1].rm_so),
match_list[1].rm_eo - match_list[1].rm_so);
buffer_terminate(description);
tmp_char = content_string[match_list[1].rm_eo];
content_string[match_list[1].rm_eo] = '\0';
if (csv_header->description)
{
free(csv_header->description);
}
csv_header->description = (char *)buffer_detach(description);
description = buffer_destroy(description);
content_string[match_list[1].rm_eo] = tmp_char;
regfree(®ex);
/* build regex for parsing CSV file lines */
if (regcomp(®ex, "^([0-9]{2}/[0-9]{2}/[0-9]{2},[0-9]{1,2}:[0-9]{2}),([0-9-]+),([0-9-]+),([0-9-]+)$", REG_EXTENDED | REG_NEWLINE))
{
goto unclean;
}
// Locate and break down CSV file lines
while (!regexec(®ex, content_string, (size_t)MAX_MATCHES, match_list, 0))
{
// Null terminate to simplify
match = match_list; match++;
timestamp = content_string + match->rm_so;
content_string[match->rm_eo] = '\0'; match++;
average = content_string + match->rm_so;
content_string[match->rm_eo] = '\0'; match++;
high = content_string + match->rm_so;
content_string[match->rm_eo] = '\0'; match++;
low = content_string + match->rm_so;
content_string[match->rm_eo] = '\0';
content_string += match->rm_eo + 1;
csv_row = csv_row_init();
if (!csv_row)
{
fprintf(stderr, "Could not allocate memory for csv row.\n");
goto unclean;
}
// Parse the timestamp in this row
memset(&time_struct, 0, sizeof(struct tm));
strptime(timestamp, "%m/%d/%y,%H:%M", &time_struct);
csv_row->timestamp = mktime(&time_struct);
csv_row->average = (int32_t)atol(average);
csv_row->high = (int32_t)atol(high);
csv_row->low = (int32_t)atol(low);
csv_row->empty = 0;
// Make sure we don't duplicate lines
if ( (csv_buffer->end_time < csv_row->timestamp) &&
((!initial_time) || (csv_row->timestamp > initial_time)) )
{
if (csv_buffer->start_time == 0)
{
csv_buffer->start_time = csv_row->timestamp;
}
// Add a new row to the list
list_append(csv_list, csv_row);
csv_buffer->end_time = csv_row->timestamp;
}
else
{
csv_row = csv_row_destroy(csv_row);
}
memset(&match_list, 0, sizeof(match_list));
}
goto clean;
unclean:
result = 0;
clean:
regfree(®ex);
description = buffer_destroy(description);
return result;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Update the csv context with the latest contents from the
* Falcon. For each buffer that has at least on hour worth of
* data, compress and write the data to the diskloop.
*/
void csv_poll( csv_context_t* csv_buffer_list, buffer_t* url_str,
st_info_t* st_info, time_t initial_time )
{
list_t* file_list = NULL;
buffer_t* buf = NULL;
buffer_t* url = NULL;
char* file_name = NULL;
const char* path = "/data/minute";
csv_buffer_t csv_tmp;
csv_buffer_t* csv_buffer;
int location = 0;
uint64_t file_hash = 0LL;
uint8_t buffer_found = 0;
int tally = 0;
// Build the CSV directory URL
url = buffer_init();
buffer_write(url, url_str->content, url_str->length);
buffer_write(url, (uint8_t*)path, strlen(path));
buffer_terminate(url);
// Initialize the CSV file list
file_list = (list_t*)malloc(sizeof(list_t));
if (!file_list)
goto clean;
list_init(file_list);
list_attributes_seeker( file_list, _file_list_seeker );
list_attributes_comparator( file_list, _file_list_comparator );
// Get the html page listing the available CSV files
buf = buffer_init();
get_page((char*)url->content, buf);
// Generate a list of files from the page
if (!csv_get_file_list(file_list, buf))
goto clean;
buffer_reset(buf);
buffer_reset(url);
// Step through each CSV file and update its csv_buffer
// in the csv_buffer_list
while (!list_empty(file_list))
{
tally++;
file_name = (char*)list_fetch(file_list);
memset(&csv_tmp, 0, sizeof(csv_tmp));
csv_tmp.file_name = file_name;
// If there is not a csv buffer for this csv file, create a
// new buffer and add it to the list
if ((location = list_locate(csv_buffer_list, &csv_tmp)) < 0)
{
csv_buffer = csv_buffer_init();
csv_buffer->file_name = file_name;
list_append(csv_buffer_list, csv_buffer);
buffer_found = 0;
// Otherwise re-use the old csv buffer
} else {
csv_buffer = (csv_buffer_t*)list_get_at(csv_buffer_list, location);
buffer_found = 1;
}
// Process the contents of this CSV file
// Generate the URL for retrieving the file
buffer_write(url, url_str->content, url_str->length);
buffer_write(url, (uint8_t*)csv_buffer->file_name,
strlen(csv_buffer->file_name));
buffer_terminate(url);
if (gDebug) {
printf("getting page %s\n", url->content);
}
// Download the file
get_page((char*)url->content, buf);
file_hash = murmur_64_b( buf->content, buf->length, HASH_SEED_64 );
if (gDebug) {
fprintf(stderr, "file '%s' [0x%016llx] uncompressed size is %lu bytes\n",
csv_buffer->file_name, (unsigned long long)file_hash,
(unsigned long)buf->length);
if (strcmp("/data/minute/logm1.csv", csv_buffer->file_name) == 0)
fprintf(stderr, "'%s'\n", buf->content);
}
// Populate a csv_buffer with the contents of the file
csv_parse_file(csv_buffer, buf, initial_time);
if (gDebug) {
fprintf(stderr, "The CSV buffer contains %u rows\n", csv_buffer->list->numels);
}
// Empty our temporary buffers
buffer_reset(buf);
buffer_reset(url);
if (buffer_found) {
free(file_name);
}
file_name = NULL;
csv_buffer = NULL;
}
// Clean up all temporary resources
clean:
buf = buffer_destroy(buf);
url = buffer_destroy(url);
if (file_list)
{
while(!list_empty(file_list))
{
file_name = list_fetch(file_list);
if (file_name)
{
free(file_name);
}
}
list_destroy(file_list);
free(file_list);
}
} // csv_poll()
int main() {
struct tm t;
time_t zero = 0;
struct buffer b[1];
buffer_init(b);
assert(b->base == NULL);
assert(b->pos == 0);
assert(b->size == 0);
buffer_putc(b, 'a');
assert(b->base != NULL);
assert(!strncmp(b->base, "a", 1));
assert(b->pos == 1);
assert(b->pos < b->size);
buffer_putc(b, 'b');
assert(b->base != NULL);
assert(!strncmp(b->base, "ab", 2));
assert(b->pos == 2);
assert(b->pos < b->size);
buffer_append(b, "12345678901234567890");
assert(b->base != NULL);
assert(!strncmp(b->base, "ab12345678901234567890", 22));
assert(b->pos == 22);
assert(b->pos < b->size);
buffer_append_n(b, "spong", 4);
assert(b->base != NULL);
assert(!strncmp(b->base, "ab12345678901234567890spon", 26));
assert(b->pos == 26);
assert(b->pos < b->size);
buffer_printf(b, "%s", "");
assert(b->base != NULL);
assert(!strncmp(b->base, "ab12345678901234567890spon", 26));
assert(b->pos == 26);
assert(b->pos < b->size);
buffer_printf(b, "%s", "123456");
assert(b->base != NULL);
assert(!strncmp(b->base, "ab12345678901234567890spon123456", 32));
assert(b->pos == 32);
assert(b->pos < b->size);
b->pos -= 6;
buffer_printf(b, "%s", "ABCDEFGHIJKLMNOPQRSTUVWXYZ");
assert(b->base != NULL);
assert(!strncmp(b->base, "ab12345678901234567890sponABCDEFGHIJKLMNOPQRSTUVWXYZ", 52));
assert(b->pos == 52);
assert(b->pos < b->size);
buffer_terminate(b);
assert(b->base != NULL);
assert(!strcmp(b->base, "ab12345678901234567890sponABCDEFGHIJKLMNOPQRSTUVWXYZ"));
assert(b->pos == 52);
assert(b->pos < b->size);
b->pos = 0;
gmtime_r(&zero, &t);
buffer_strftime(b, "", &t);
assert(b->pos == 0);
buffer_strftime(b, "%F %T", &t);
buffer_terminate(b);
assert(!strcmp(b->base, "1970-01-01 00:00:00"));
free(b->base);
return 0;
}
