IV
PerlIOMmap_map(pTHX_ PerlIO *f)
{
dVAR;
PerlIOMmap * const m = PerlIOSelf(f, PerlIOMmap);
const IV flags = PerlIOBase(f)->flags;
IV code = 0;
if (m->len)
abort();
if (flags & PERLIO_F_CANREAD) {
PerlIOBuf * const b = PerlIOSelf(f, PerlIOBuf);
const int fd = PerlIO_fileno(f);
Stat_t st;
code = Fstat(fd, &st);
if (code == 0 && S_ISREG(st.st_mode)) {
SSize_t len = st.st_size - b->posn;
if (len > 0) {
Off_t posn;
if (PL_mmap_page_size <= 0)
Perl_croak(aTHX_ "panic: bad pagesize %" IVdf,
PL_mmap_page_size);
if (b->posn < 0) {
/*
* This is a hack - should never happen - open should
* have set it !
*/
b->posn = PerlIO_tell(PerlIONext(f));
}
posn = (b->posn / PL_mmap_page_size) * PL_mmap_page_size;
len = st.st_size - posn;
m->mptr = (Mmap_t)mmap(NULL, len, PROT_READ, MAP_SHARED, fd, posn);
if (m->mptr && m->mptr != (Mmap_t) - 1) {
#if 0 && defined(HAS_MADVISE) && defined(MADV_SEQUENTIAL)
madvise(m->mptr, len, MADV_SEQUENTIAL);
#endif
#if 0 && defined(HAS_MADVISE) && defined(MADV_WILLNEED)
madvise(m->mptr, len, MADV_WILLNEED);
#endif
PerlIOBase(f)->flags =
(flags & ~PERLIO_F_EOF) | PERLIO_F_RDBUF;
b->end = ((STDCHAR *) m->mptr) + len;
b->buf = ((STDCHAR *) m->mptr) + (b->posn - posn);
b->ptr = b->buf;
m->len = len;
}
else {
b->buf = NULL;
}
}
else {
PerlIOBase(f)->flags =
flags | PERLIO_F_EOF | PERLIO_F_RDBUF;
b->buf = NULL;
b->ptr = b->end = b->ptr;
code = -1;
}
}
}
return code;
}
Off_t
PerlIOEncode_tell(pTHX_ PerlIO * f)
{
PerlIOBuf *b = PerlIOSelf(f, PerlIOBuf);
/* Unfortunately the only way to get a postion is to (re-)translate,
the UTF8 we have in bufefr and then ask layer below
*/
PerlIO_flush(f);
if (b->buf && b->ptr > b->buf) {
Perl_croak(aTHX_ "Cannot tell at partial character");
}
return PerlIO_tell(PerlIONext(f));
}
void
_wavpack_skip(wvpinfo *wvp, uint32_t size)
{
if ( buffer_len(wvp->buf) >= size ) {
//buffer_dump(mp4->buf, size);
buffer_consume(wvp->buf, size);
DEBUG_TRACE(" skipped buffer data size %d\n", size);
}
else {
PerlIO_seek(wvp->infile, size - buffer_len(wvp->buf), SEEK_CUR);
buffer_clear(wvp->buf);
DEBUG_TRACE(" seeked past %d bytes to %d\n", size, (int)PerlIO_tell(wvp->infile));
}
}
int csv_select_run_child(
csv_parse_t *parse, csv_result_set_t *result, size_t table_pos
) {
csv_select_t *select = parse->select;
csv_table_def_t *table, *table2;
char *s1, *s2;
int ch, has_child = select->table_count - table_pos > 1;
size_t i, j, k, l, m, rp = 0, nr = 0, l1;
Expr *x1;
ExprList *xl_cols = select->columns;
csv_row_t *row;
csv_var_t *v1, v2;
csv_column_def_t *col;
csv_t *csv = parse->csv;
/* reset table */
table = select->tables[table_pos];
PerlIO_seek( table->dstream, table->header_offset, SEEK_SET );
if( table->flags & TEXT_TABLE_COLNAMEHEADER ) {
if( ! table->header_offset ) {
do {
ch = PerlIO_getc( table->dstream );
} while( ch != EOF && ch != '\n' );
table->header_offset = PerlIO_tell( table->dstream );
}
}
table->row_pos = 0;
/* read rows */
while( (s1 = csv_read_row( table )) != NULL ) {
#ifdef CSV_DEBUG
_debug( "table '%s' row %lu\n", table->name, table->row_pos );
#endif
if( has_child ) {
ch = csv_select_run_child( parse, result, table_pos + 1 );
if( ch != CSV_OK )
return ch;
continue;
}
/* eval WHERE */
if( (x1 = select->where) != NULL ) {
ch = expr_eval( parse, x1 );
if( ch != CSV_OK )
return ch;
if( x1->var.flags & VAR_HAS_IV ) {
if( ! x1->var.iv )
continue;
}
else if( x1->var.flags & VAR_HAS_NV ) {
if( ! x1->var.nv )
continue;
}
else if( x1->var.sv == NULL || x1->var.sv[0] == '0' )
continue;
}
/* eval columns */
for( i = 0; i < xl_cols->expr_count; i ++ ) {
x1 = xl_cols->expr[i];
if( (x1->flags & EXPR_IS_AGGR) == 0 ) {
ch = expr_eval( parse, x1 );
if( ch != CSV_OK )
return ch;
}
}
if( select->groupby == NULL )
goto add_row;
/* eval groupby */
for( i = 0; i < select->groupby->expr_count; i ++ ) {
ch = expr_eval( parse, select->groupby->expr[i] );
if( ch != CSV_OK )
return ch;
}
/* search groupby row */
for( j = 0; j < result->row_count; j ++ ) {
row = result->rows[j];
for( i = 0; i < select->groupby->expr_count; i ++ ) {
v1 = &row->groupby[i];
x1 = select->groupby->expr[i];
VAR_COMP_OP( v2, *v1, x1->var, ==, "==", 0, 0 );
if( ! v2.iv )
goto groupby_next;
}
goto eval_row;
groupby_next:
continue;
}
add_row:
if( select->offset && rp < select->offset ) {
rp ++;
continue;
}
if( select->limit && rp - select->offset >= select->limit )
goto finish;
rp ++;
Newxz( row, 1, csv_row_t );
row->select = select;
if( (result->row_count % ROW_COUNT_EXPAND) == 0 ) {
Renew( result->rows, result->row_count + ROW_COUNT_EXPAND,
csv_row_t * );
}
result->rows[result->row_count ++] = row;
Newxz( row->data, result->column_count, csv_var_t );
if( select->orderby != NULL )
Newxz( row->orderby, select->orderby->expr_count, csv_var_t );
if( select->groupby != NULL )
Newxz( row->groupby, select->groupby->expr_count, csv_var_t );
nr = 1;
eval_row:
for( i = 0, j = 0; i < xl_cols->expr_count; i ++, j ++ ) {
x1 = xl_cols->expr[i];
if( x1->op == TK_TABLE ) {
table2 = x1->table;
for( l = 0; l < table2->column_count; l ++, j ++ ) {
col = table2->columns + l;
v1 = row->data + j;
s1 = table->data + col->offset;
l1 = col->length;
if( l1 == 4 && strcmp( s1, "NULL" ) == 0 )
continue;
switch( col->type ) {
case FIELD_TYPE_INTEGER:
v1->iv = atoi( s1 );
v1->flags = VAR_HAS_IV;
break;
case FIELD_TYPE_DOUBLE:
if( table2->decimal_symbol != '.' ) {
s2 = strchr( s1, table2->decimal_symbol );
if( s2 != NULL )
*s2 = '.';
}
v1->nv = atof( s1 );
v1->flags = VAR_HAS_NV;
break;
case FIELD_TYPE_CHAR:
v1->sv_len = charset_convert(
s1, l1, table2->charset,
csv->client_charset, &v1->sv
);
v1->flags = VAR_HAS_SV;
break;
case FIELD_TYPE_BLOB:
if( l1 >= 2 && s1[0] == '0' && s1[1] == 'x' ) {
l1 = (l1 - 2) / 2;
if( ! l1 ) {
v1->flags = 0;
continue;
}
Renew( v1->sv, l1 + 1, char );
for( m = 0, s1 += 2; m < l1; m ++ ) {
ch = CHAR_FROM_HEX[*s1 & 0x7f] << 4;
s1 ++;
ch += CHAR_FROM_HEX[*s1 & 0x7f];
s1 ++;
v1->sv[m] = (char) ch;
}
v1->sv[l1] = '\0';
v1->sv_len = l1;
v1->flags = VAR_HAS_SV;
break;
}
default:
Renew( v1->sv, l1 + 1, char );
Copy( s1, v1->sv, l1, char );
v1->sv[l1] = '\0';
v1->sv_len = l1;
v1->flags = VAR_HAS_SV;
break;
}
}
continue;
}
PerlIO_getpos(PerlIO *f, Fpos_t *pos)
{
*pos = PerlIO_tell(f);
return 0;
}
SV * parse_in_chunks(char * filepath, size_t filesize) {
char *buf;
size_t bytes_read = 0;
int max_buf = 1000;
char *err_msg;
int block = BLOCK_HEADER;
int cur_event_type = 0;
int event_type = 0;
char event_block = 0;
char *brnl, *breq;
AV * data;
AV * datawrapper;
AV * events;
char *line;
char * nl = "\n";
char * eq = "=";
int rewind_pos = 0;
size_t cur_fpos = 0;
SV * pbuf;
SV * pmax_buf;
AV * HANDLERS = get_av("Opsview::Utils::NDOLogsImporter::HANDLERS", 0);
AV * INPUT_DATA_TYPE = get_av("Opsview::Utils::NDOLogsImporter::INPUT_DATA_TYPE", 0);
int init_last_pos;
int init_block;
if ( first_read ) {
if ( ! ( fh = PerlIO_open( filepath, "rb" ) ) ) {
croak("Could not open file: %s\n", strerror(errno));
}
bytes_left = filesize;
init_last_pos = prev_pos = first_read = 0;
init_block = block = BLOCK_HEADER;
} else {
init_block = block = BLOCK_EVENTS;
init_last_pos = prev_pos;
}
read_begin:
brnl = NULL;
breq = NULL;
pbuf = get_sv("Opsview::Utils::NDOLogsImporter::PARSE_BUF", 0);
pmax_buf = get_sv("Opsview::Utils::NDOLogsImporter::MAX_BUF_SIZE", 0);
buf = SvPVX(pbuf);
max_buf = SvIVX(pmax_buf);
if ( max_buf < 1024 * 1024 && ! automated_tests ) {
max_buf = 1024*1024;
SvIV_set( pmax_buf, max_buf );
SvGROW( pbuf, max_buf + 1);
SvCUR_set( pbuf, max_buf);
}
if ( bytes_left > 0 ) {
bytes_read = PerlIO_read(fh, buf + prev_pos, max_buf-prev_pos);
cur_fpos = PerlIO_tell(fh);
if ( bytes_read < 0 ) {
err_msg = strerror(errno);
PerlIO_close( fh );
croak("Could not read file: %s\n", err_msg);
}
bytes_left -= bytes_read;
events = (AV *)sv_2mortal((SV *)newAV());
rewind_pos = last_999(buf+prev_pos, bytes_read);
prev_pos = bytes_read + prev_pos - rewind_pos;
buf[prev_pos] = '\0';
// avg ratio events:file_size = 0.21%
if ( prev_pos > 1000 ) {
av_extend( events, (int)(prev_pos * 0.0021) );
}
for ( line = strtok_r(buf, nl, &brnl); line != NULL; line = strtok_r(NULL, nl, &brnl) )
{
switch(block) {
case BLOCK_HEADER:
{
if ( strEQ(line, "STARTDATADUMP") ) {
block = BLOCK_EVENTS;
}
}
break;
case BLOCK_EVENTS:
{
if ( strEQ(line, "1000") ) { /* NDO_API_ENDDATADUMP */
block = BLOCK_FOOTER;
continue;
}
cur_event_type = atoi(line);
/* ignore events we are not handling */
if ( ! av_exists(HANDLERS, cur_event_type) ) {
block = BLOCK_IGNORE_EVENT;
continue;
}
event_block = BLOCK_EVENT_STARTED;
if ( cur_event_type != event_type ) {
datawrapper = (AV *)sv_2mortal((SV *)newAV());
data = (AV *)sv_2mortal((SV *)newAV());
av_push( events, newSViv( cur_event_type ) );
av_push( datawrapper, newRV( (SV *)data ) );
av_push( events, newRV( (SV *)datawrapper ) );
event_type = cur_event_type;
} else {
data = (AV *)sv_2mortal((SV *)newAV());
av_push( datawrapper, newRV( (SV *)data ) );
}
block = BLOCK_EVENT;
}
break;
case BLOCK_EVENT:
{
if ( strEQ(line, "999") ) { /* NDO_API_ENDDATA */
block = BLOCK_EVENTS;
event_block = BLOCK_EVENT_ENDED;
} else {
char *k;
char *v;
int key;
int key_type = 0;
int v_len = 0;
k = strtok_r(line, eq, &breq);
v = strtok_r(NULL, "\0", &breq);
key = atoi(k);
/* invalid key, skip parsing */
if ( key == 0 ) {
goto remove_invalid;
}
SV ** const k_type = av_fetch(INPUT_DATA_TYPE, key, 0 );
if ( k_type ) {
key_type = SvIVx( *k_type );
}
if ( v ) {
if ( key_type & 1 ) {
v_len = ndo_unescape_buffer( v );
} else {
v_len = strlen(v);
}
}
if ( key_type & 2 ) {
AV * datanstptr;
SV ** const datanst = av_fetch(data, key, 0 );
if ( datanst ) {
datanstptr = (AV *)SvRV( *datanst );
} else {
datanstptr = (AV *)sv_2mortal((SV *)newAV());
av_store( data, key, newRV( (SV *)datanstptr ) );
}
if ( v ) {
av_push( datanstptr, newSVpvn(v, v_len) );
} else {
av_push( datanstptr, newSVpvn("", 0) );
}
} else {
if ( v ) {
av_store( data, key, newSVpvn(v, v_len) );
} else {
av_store( data, key, newSVpvn("", 0) );
}
}
}
}
break;
case BLOCK_FOOTER:
{
if ( strEQ(line, "GOODBYE") ) {
block = BLOCK_HEADER;
}
}
break;
case BLOCK_IGNORE_EVENT:
{
if ( strEQ(line, "999") ) { /* NDO_API_ENDDATA */
block = BLOCK_EVENTS; // go back to EVENTS
continue;
}
}
break;
}
};
/* there were some events */
if ( event_block != BLOCK_HEADER ) {
if ( event_block != BLOCK_EVENT_ENDED ) {
remove_invalid:
av_pop( datawrapper );
}
/* remove whole block if the last block has no events */
if ( av_len( datawrapper ) == -1 ) {
av_pop( events );
av_pop( events );
}
}
if ( av_len(events) > 0 ) {
if ( rewind_pos > 0 && cur_fpos < filesize ) {
memmove(buf, buf+prev_pos+1, rewind_pos-1);
}
prev_pos = rewind_pos - 1;
return newRV_inc((SV *) events);
} else {
if ( cur_fpos < filesize && event_block != BLOCK_HEADER && event_block != BLOCK_EVENT_ENDED ) {
int new_max_buf = max_buf * 2;
SvIV_set( pmax_buf, new_max_buf );
SvGROW( pbuf, new_max_buf + 1);
SvCUR_set( pbuf, new_max_buf);
//start again as previous buffer would be tokenized already
prev_pos = 0;
block = init_block;
event_type = 0;
PerlIO_close( fh );
if ( ! ( fh = PerlIO_open( filepath, "rb" ) ) ) {
croak("Could not re-open file: %s\n", strerror(errno));
}
PerlIO_seek(fh, cur_fpos-bytes_read-init_last_pos, SEEK_SET);
bytes_left += bytes_read + init_last_pos;
goto read_begin;
}
}
}
parser_reset_iterator();
return &PL_sv_undef;
}
int
_check_buf(PerlIO *infile, Buffer *buf, int min_wanted, int max_wanted)
{
int ret = 1;
// Do we have enough data?
if ( buffer_len(buf) < min_wanted ) {
// Read more data
uint32_t read;
uint32_t actual_wanted;
unsigned char *tmp;
#ifdef _MSC_VER
uint32_t pos_check = PerlIO_tell(infile);
#endif
if (min_wanted > max_wanted) {
max_wanted = min_wanted;
}
// Adjust actual amount to read by the amount we already have in the buffer
actual_wanted = max_wanted - buffer_len(buf);
New(0, tmp, actual_wanted, unsigned char);
DEBUG_TRACE("Buffering from file @ %d (min_wanted %d, max_wanted %d, adjusted to %d)\n",
(int)PerlIO_tell(infile), min_wanted, max_wanted, actual_wanted
);
if ( (read = PerlIO_read(infile, tmp, actual_wanted)) <= 0 ) {
if ( PerlIO_error(infile) ) {
#ifdef _MSC_VER
// Show windows specific error message as Win32 PerlIO_read does not set errno
DWORD last_error = GetLastError();
LPWSTR *errmsg = NULL;
FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, 0, last_error, 0, (LPWSTR)&errmsg, 0, NULL);
warn("Error reading: %d %s (read %d wanted %d)\n", last_error, errmsg, read, actual_wanted);
LocalFree(errmsg);
#else
warn("Error reading: %s (wanted %d)\n", strerror(errno), actual_wanted);
#endif
}
else {
warn("Error: Unable to read at least %d bytes from file.\n", min_wanted);
}
ret = 0;
goto out;
}
buffer_append(buf, tmp, read);
// Make sure we got enough
if ( buffer_len(buf) < min_wanted ) {
warn("Error: Unable to read at least %d bytes from file (only read %d).\n", min_wanted, read);
ret = 0;
goto out;
}
#ifdef _MSC_VER
// Bug 16095, weird off-by-one bug seen only on Win32 and only when reading a filehandle
if (PerlIO_tell(infile) != pos_check + read) {
//PerlIO_printf(PerlIO_stderr(), "Win32 bug, pos should be %d, but was %d\n", pos_check + read, PerlIO_tell(infile));
PerlIO_seek(infile, pos_check + read, SEEK_SET);
}
#endif
DEBUG_TRACE("Buffered %d bytes, new pos %d\n", read, (int)PerlIO_tell(infile));
out:
Safefree(tmp);
}
