static int
fmm_mime_magic(PerlFMM *state, char *file, char **mime_type)
{
PerlIO *fhandle;
SV *err;
int ret;
if ((ret = fmm_fsmagic(state, file, mime_type)) == 0) {
return 0;
}
if (ret == -1) {
return -1;
}
fhandle = PerlIO_open(file, "r");
if (!fhandle) {
err = newSVpvf(
"Failed to open file %s: %s", file, strerror(errno));
FMM_SET_ERROR(state, err);
return -1;
}
if ((ret = fmm_fhmagic(state, fhandle, mime_type)) == 0) {
#ifdef FMM_DEBUG
PerlIO_printf(PerlIO_stderr(), "[fmm_mime_magic]: fmm_fhmagic returns 0\n");
#endif
PerlIO_close(fhandle);
return 0;
}
PerlIO_close(fhandle);
return fmm_ext_magic(state, file, mime_type);
}
/* maps to mod_mime_magic::apprentice */
static int
fmm_parse_magic_file(PerlFMM *state, char *file)
{
int ws_offset;
int lineno;
int errs;
/* char line[BUFSIZ + 1];*/
PerlIO *fhandle;
SV *err;
SV *sv = sv_2mortal(newSV(BUFSIZ));
SV *PL_rs_orig = newSVsv(PL_rs);
char *line;
fhandle = PerlIO_open(file, "r");
if (! fhandle) {
err = newSVpvf(
"Failed to open %s: %s", file, strerror(errno));
FMM_SET_ERROR(state, err);
PerlIO_close(fhandle);
return -1;
}
/*
* Parse it line by line
* $/ (slurp mode) is needed here
*/
PL_rs = sv_2mortal(newSVpvn("\n", 1));
for(lineno = 1; sv_gets(sv, fhandle, 0) != NULL; lineno++) {
line = SvPV_nolen(sv);
/* delete newline */
if (line[0]) {
line[strlen(line) - 1] = '\0';
}
/* skip leading whitespace */
ws_offset = 0;
while (line[ws_offset] && isSPACE(line[ws_offset])) {
ws_offset++;
}
/* skip blank lines */
if (line[ws_offset] == 0) {
continue;
}
if (line[ws_offset] == '#') {
continue;
}
if (fmm_parse_magic_line(state, line, lineno) != 0) {
++errs;
}
}
PerlIO_close(fhandle);
PL_rs = PL_rs_orig;
return 1;
}
void csv_table_def_free( csv_table_def_t *table ) {
size_t i;
if( table == NULL )
return;
if( table->alias != table->name )
Safefree( table->alias );
Safefree( table->name );
for( i = 0; i < table->column_count; i ++ )
csv_column_def_free( &table->columns[i] );
Safefree( table->columns );
if( table->dstream != NULL )
PerlIO_close( table->dstream );
Safefree( table->data );
Safefree( table );
}
STATIC char *
S_skipspace(pTHX_ register char *s, int incline)
{
if (PL_lex_formbrack && PL_lex_brackets <= PL_lex_formbrack) {
while (s < PL_bufend && SPACE_OR_TAB(*s))
s++;
return s;
}
for (;;) {
STRLEN prevlen;
SSize_t oldprevlen, oldoldprevlen;
SSize_t oldloplen = 0, oldunilen = 0;
while (s < PL_bufend && isSPACE(*s)) {
if (*s++ == '\n' && ((incline == 2) || (PL_in_eval && !PL_rsfp && !incline)))
incline(s);
}
/* comment */
if (s < PL_bufend && *s == '#') {
while (s < PL_bufend && *s != '\n')
s++;
if (s < PL_bufend) {
s++;
if (PL_in_eval && !PL_rsfp && !incline) {
incline(s);
continue;
}
}
}
/* also skip leading whitespace on the beginning of a line before deciding
* whether or not to recharge the linestr. --rafl
*/
while (s < PL_bufend && isSPACE(*s)) {
if (*s++ == '\n' && PL_in_eval && !PL_rsfp && !incline)
incline(s);
}
/* only continue to recharge the buffer if we're at the end
* of the buffer, we're not reading from a source filter, and
* we're in normal lexing mode
*/
if (s < PL_bufend || !PL_rsfp || PL_lex_inwhat ||
PL_lex_state == LEX_FORMLINE)
return s;
/* try to recharge the buffer */
if ((s = filter_gets(PL_linestr, PL_rsfp,
(prevlen = SvCUR(PL_linestr)))) == Nullch)
{
/* end of file. Add on the -p or -n magic */
if (PL_minus_p) {
sv_setpv(PL_linestr,
";}continue{print or die qq(-p destination: $!\\n);}");
PL_minus_n = PL_minus_p = 0;
}
else if (PL_minus_n) {
sv_setpvn(PL_linestr, ";}", 2);
PL_minus_n = 0;
}
else
sv_setpvn(PL_linestr,";", 1);
/* reset variables for next time we lex */
PL_oldoldbufptr = PL_oldbufptr = PL_bufptr = s = PL_linestart
= SvPVX(PL_linestr);
PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
PL_last_lop = PL_last_uni = Nullch;
/* In perl versions previous to p4-rawid: //depot/perl@32954 -P
* preprocessors were supported here. We don't support -P at all, even
* on perls that support it, and use the following chunk from blead
* perl. (rafl)
*/
/* Close the filehandle. Could be from
* STDIN, or a regular file. If we were reading code from
* STDIN (because the commandline held no -e or filename)
* then we don't close it, we reset it so the code can
* read from STDIN too.
*/
if ((PerlIO*)PL_rsfp == PerlIO_stdin())
PerlIO_clearerr(PL_rsfp);
else
(void)PerlIO_close(PL_rsfp);
PL_rsfp = Nullfp;
return s;
}
/* not at end of file, so we only read another line */
/* make corresponding updates to old pointers, for yyerror() */
oldprevlen = PL_oldbufptr - PL_bufend;
oldoldprevlen = PL_oldoldbufptr - PL_bufend;
if (PL_last_uni)
oldunilen = PL_last_uni - PL_bufend;
if (PL_last_lop)
oldloplen = PL_last_lop - PL_bufend;
PL_linestart = PL_bufptr = s + prevlen;
PL_bufend = s + SvCUR(PL_linestr);
s = PL_bufptr;
PL_oldbufptr = s + oldprevlen;
PL_oldoldbufptr = s + oldoldprevlen;
if (PL_last_uni)
PL_last_uni = s + oldunilen;
if (PL_last_lop)
PL_last_lop = s + oldloplen;
if (!incline)
incline(s);
/* debugger active and we're not compiling the debugger code,
* so store the line into the debugger's array of lines
*/
if (PERLDB_LINE && PL_curstash != PL_debstash) {
AV *fileav = CopFILEAV(PL_curcop);
if (fileav) {
SV * const sv = NEWSV(85,0);
sv_upgrade(sv, SVt_PVMG);
sv_setpvn(sv,PL_bufptr,PL_bufend-PL_bufptr);
(void)SvIOK_on(sv);
SvIV_set(sv, 0);
av_store(fileav,(I32)CopLINE(PL_curcop),sv);
}
}
}
}
SV * parse_in_chunks(char * filepath, size_t filesize) {
char *buf;
size_t bytes_read = 0;
int max_buf = 1000;
char *err_msg;
int block = BLOCK_HEADER;
int cur_event_type = 0;
int event_type = 0;
char event_block = 0;
char *brnl, *breq;
AV * data;
AV * datawrapper;
AV * events;
char *line;
char * nl = "\n";
char * eq = "=";
int rewind_pos = 0;
size_t cur_fpos = 0;
SV * pbuf;
SV * pmax_buf;
AV * HANDLERS = get_av("Opsview::Utils::NDOLogsImporter::HANDLERS", 0);
AV * INPUT_DATA_TYPE = get_av("Opsview::Utils::NDOLogsImporter::INPUT_DATA_TYPE", 0);
int init_last_pos;
int init_block;
if ( first_read ) {
if ( ! ( fh = PerlIO_open( filepath, "rb" ) ) ) {
croak("Could not open file: %s\n", strerror(errno));
}
bytes_left = filesize;
init_last_pos = prev_pos = first_read = 0;
init_block = block = BLOCK_HEADER;
} else {
init_block = block = BLOCK_EVENTS;
init_last_pos = prev_pos;
}
read_begin:
brnl = NULL;
breq = NULL;
pbuf = get_sv("Opsview::Utils::NDOLogsImporter::PARSE_BUF", 0);
pmax_buf = get_sv("Opsview::Utils::NDOLogsImporter::MAX_BUF_SIZE", 0);
buf = SvPVX(pbuf);
max_buf = SvIVX(pmax_buf);
if ( max_buf < 1024 * 1024 && ! automated_tests ) {
max_buf = 1024*1024;
SvIV_set( pmax_buf, max_buf );
SvGROW( pbuf, max_buf + 1);
SvCUR_set( pbuf, max_buf);
}
if ( bytes_left > 0 ) {
bytes_read = PerlIO_read(fh, buf + prev_pos, max_buf-prev_pos);
cur_fpos = PerlIO_tell(fh);
if ( bytes_read < 0 ) {
err_msg = strerror(errno);
PerlIO_close( fh );
croak("Could not read file: %s\n", err_msg);
}
bytes_left -= bytes_read;
events = (AV *)sv_2mortal((SV *)newAV());
rewind_pos = last_999(buf+prev_pos, bytes_read);
prev_pos = bytes_read + prev_pos - rewind_pos;
buf[prev_pos] = '\0';
// avg ratio events:file_size = 0.21%
if ( prev_pos > 1000 ) {
av_extend( events, (int)(prev_pos * 0.0021) );
}
for ( line = strtok_r(buf, nl, &brnl); line != NULL; line = strtok_r(NULL, nl, &brnl) )
{
switch(block) {
case BLOCK_HEADER:
{
if ( strEQ(line, "STARTDATADUMP") ) {
block = BLOCK_EVENTS;
}
}
break;
case BLOCK_EVENTS:
{
if ( strEQ(line, "1000") ) { /* NDO_API_ENDDATADUMP */
block = BLOCK_FOOTER;
continue;
}
cur_event_type = atoi(line);
/* ignore events we are not handling */
if ( ! av_exists(HANDLERS, cur_event_type) ) {
block = BLOCK_IGNORE_EVENT;
continue;
}
event_block = BLOCK_EVENT_STARTED;
if ( cur_event_type != event_type ) {
datawrapper = (AV *)sv_2mortal((SV *)newAV());
data = (AV *)sv_2mortal((SV *)newAV());
av_push( events, newSViv( cur_event_type ) );
av_push( datawrapper, newRV( (SV *)data ) );
av_push( events, newRV( (SV *)datawrapper ) );
event_type = cur_event_type;
} else {
data = (AV *)sv_2mortal((SV *)newAV());
av_push( datawrapper, newRV( (SV *)data ) );
}
block = BLOCK_EVENT;
}
break;
case BLOCK_EVENT:
{
if ( strEQ(line, "999") ) { /* NDO_API_ENDDATA */
block = BLOCK_EVENTS;
event_block = BLOCK_EVENT_ENDED;
} else {
char *k;
char *v;
int key;
int key_type = 0;
int v_len = 0;
k = strtok_r(line, eq, &breq);
v = strtok_r(NULL, "\0", &breq);
key = atoi(k);
/* invalid key, skip parsing */
if ( key == 0 ) {
goto remove_invalid;
}
SV ** const k_type = av_fetch(INPUT_DATA_TYPE, key, 0 );
if ( k_type ) {
key_type = SvIVx( *k_type );
}
if ( v ) {
if ( key_type & 1 ) {
v_len = ndo_unescape_buffer( v );
} else {
v_len = strlen(v);
}
}
if ( key_type & 2 ) {
AV * datanstptr;
SV ** const datanst = av_fetch(data, key, 0 );
if ( datanst ) {
datanstptr = (AV *)SvRV( *datanst );
} else {
datanstptr = (AV *)sv_2mortal((SV *)newAV());
av_store( data, key, newRV( (SV *)datanstptr ) );
}
if ( v ) {
av_push( datanstptr, newSVpvn(v, v_len) );
} else {
av_push( datanstptr, newSVpvn("", 0) );
}
} else {
if ( v ) {
av_store( data, key, newSVpvn(v, v_len) );
} else {
av_store( data, key, newSVpvn("", 0) );
}
}
}
}
break;
case BLOCK_FOOTER:
{
if ( strEQ(line, "GOODBYE") ) {
block = BLOCK_HEADER;
}
}
break;
case BLOCK_IGNORE_EVENT:
{
if ( strEQ(line, "999") ) { /* NDO_API_ENDDATA */
block = BLOCK_EVENTS; // go back to EVENTS
continue;
}
}
break;
}
};
/* there were some events */
if ( event_block != BLOCK_HEADER ) {
if ( event_block != BLOCK_EVENT_ENDED ) {
remove_invalid:
av_pop( datawrapper );
}
/* remove whole block if the last block has no events */
if ( av_len( datawrapper ) == -1 ) {
av_pop( events );
av_pop( events );
}
}
if ( av_len(events) > 0 ) {
if ( rewind_pos > 0 && cur_fpos < filesize ) {
memmove(buf, buf+prev_pos+1, rewind_pos-1);
}
prev_pos = rewind_pos - 1;
return newRV_inc((SV *) events);
} else {
if ( cur_fpos < filesize && event_block != BLOCK_HEADER && event_block != BLOCK_EVENT_ENDED ) {
int new_max_buf = max_buf * 2;
SvIV_set( pmax_buf, new_max_buf );
SvGROW( pbuf, new_max_buf + 1);
SvCUR_set( pbuf, new_max_buf);
//start again as previous buffer would be tokenized already
prev_pos = 0;
block = init_block;
event_type = 0;
PerlIO_close( fh );
if ( ! ( fh = PerlIO_open( filepath, "rb" ) ) ) {
croak("Could not re-open file: %s\n", strerror(errno));
}
PerlIO_seek(fh, cur_fpos-bytes_read-init_last_pos, SEEK_SET);
bytes_left += bytes_read + init_last_pos;
goto read_begin;
}
}
}
parser_reset_iterator();
return &PL_sv_undef;
}
void parser_reset_iterator() {
first_read = 1;
prev_pos = 0;
bytes_left = 0;
if (fh) PerlIO_close( fh );
}
