Ejemplo n.º 1
0
IV
PerlIOMmap_map(pTHX_ PerlIO *f)
{
    dVAR;
    PerlIOMmap * const m = PerlIOSelf(f, PerlIOMmap);
    const IV flags = PerlIOBase(f)->flags;
    IV code = 0;
    if (m->len)
	abort();
    if (flags & PERLIO_F_CANREAD) {
	PerlIOBuf * const b = PerlIOSelf(f, PerlIOBuf);
	const int fd = PerlIO_fileno(f);
	Stat_t st;
	code = Fstat(fd, &st);
	if (code == 0 && S_ISREG(st.st_mode)) {
	    SSize_t len = st.st_size - b->posn;
	    if (len > 0) {
		Off_t posn;
		if (PL_mmap_page_size <= 0)
		  Perl_croak(aTHX_ "panic: bad pagesize %" IVdf,
			     PL_mmap_page_size);
		if (b->posn < 0) {
		    /*
		     * This is a hack - should never happen - open should
		     * have set it !
		     */
		    b->posn = PerlIO_tell(PerlIONext(f));
		}
		posn = (b->posn / PL_mmap_page_size) * PL_mmap_page_size;
		len = st.st_size - posn;
		m->mptr = (Mmap_t)mmap(NULL, len, PROT_READ, MAP_SHARED, fd, posn);
		if (m->mptr && m->mptr != (Mmap_t) - 1) {
#if 0 && defined(HAS_MADVISE) && defined(MADV_SEQUENTIAL)
		    madvise(m->mptr, len, MADV_SEQUENTIAL);
#endif
#if 0 && defined(HAS_MADVISE) && defined(MADV_WILLNEED)
		    madvise(m->mptr, len, MADV_WILLNEED);
#endif
		    PerlIOBase(f)->flags =
			(flags & ~PERLIO_F_EOF) | PERLIO_F_RDBUF;
		    b->end = ((STDCHAR *) m->mptr) + len;
		    b->buf = ((STDCHAR *) m->mptr) + (b->posn - posn);
		    b->ptr = b->buf;
		    m->len = len;
		}
		else {
		    b->buf = NULL;
		}
	    }
	    else {
		PerlIOBase(f)->flags =
		    flags | PERLIO_F_EOF | PERLIO_F_RDBUF;
		b->buf = NULL;
		b->ptr = b->end = b->ptr;
		code = -1;
	    }
	}
    }
    return code;
}
Ejemplo n.º 2
0
Off_t
PerlIOEncode_tell(pTHX_ PerlIO * f)
{
    PerlIOBuf *b = PerlIOSelf(f, PerlIOBuf);
    /* Unfortunately the only way to get a postion is to (re-)translate,
       the UTF8 we have in bufefr and then ask layer below
     */
    PerlIO_flush(f);
    if (b->buf && b->ptr > b->buf) {
	Perl_croak(aTHX_ "Cannot tell at partial character");
    }
    return PerlIO_tell(PerlIONext(f));
}
Ejemplo n.º 3
0
void
_wavpack_skip(wvpinfo *wvp, uint32_t size)
{
  if ( buffer_len(wvp->buf) >= size ) {
    //buffer_dump(mp4->buf, size);
    buffer_consume(wvp->buf, size);
    
    DEBUG_TRACE("  skipped buffer data size %d\n", size);
  }
  else {
    PerlIO_seek(wvp->infile, size - buffer_len(wvp->buf), SEEK_CUR);
    buffer_clear(wvp->buf);
    
    DEBUG_TRACE("  seeked past %d bytes to %d\n", size, (int)PerlIO_tell(wvp->infile));
  }
}
Ejemplo n.º 4
0
int csv_select_run_child(
	csv_parse_t *parse, csv_result_set_t *result, size_t table_pos
) {
	csv_select_t *select = parse->select;
	csv_table_def_t *table, *table2;
	char *s1, *s2;
	int ch, has_child = select->table_count - table_pos > 1;
	size_t i, j, k, l, m, rp = 0, nr = 0, l1;
	Expr *x1;
	ExprList *xl_cols = select->columns;
	csv_row_t *row;
	csv_var_t *v1, v2;
	csv_column_def_t *col;
	csv_t *csv = parse->csv;
	/* reset table */
	table = select->tables[table_pos];
	PerlIO_seek( table->dstream, table->header_offset, SEEK_SET );
	if( table->flags & TEXT_TABLE_COLNAMEHEADER ) {
		if( ! table->header_offset ) {
			do {
				ch = PerlIO_getc( table->dstream );
			} while( ch != EOF && ch != '\n' );
			table->header_offset = PerlIO_tell( table->dstream );
		}
	}
	table->row_pos = 0;
	/* read rows */
	while( (s1 = csv_read_row( table )) != NULL ) {
#ifdef CSV_DEBUG
		_debug( "table '%s' row %lu\n", table->name, table->row_pos );
#endif
		if( has_child ) {
			ch = csv_select_run_child( parse, result, table_pos + 1 );
			if( ch != CSV_OK )
				return ch;
			continue;
		}
		/* eval WHERE */
		if( (x1 = select->where) != NULL ) {
			ch = expr_eval( parse, x1 );
			if( ch != CSV_OK )
				return ch;
			if( x1->var.flags & VAR_HAS_IV ) {
				if( ! x1->var.iv )
					continue;
			}
			else if( x1->var.flags & VAR_HAS_NV ) {
				if( ! x1->var.nv )
					continue;
			}
			else if( x1->var.sv == NULL || x1->var.sv[0] == '0' )
				continue;
		}
		/* eval columns */
		for( i = 0; i < xl_cols->expr_count; i ++ ) {
			x1 = xl_cols->expr[i];
			if( (x1->flags & EXPR_IS_AGGR) == 0 ) {
				ch = expr_eval( parse, x1 );
				if( ch != CSV_OK )
					return ch;
			}
		}
		if( select->groupby == NULL )
			goto add_row;
		/* eval groupby */
		for( i = 0; i < select->groupby->expr_count; i ++ ) {
			ch = expr_eval( parse, select->groupby->expr[i] );
			if( ch != CSV_OK )
				return ch;
		}
		/* search groupby row */
		for( j = 0; j < result->row_count; j ++ ) {
			row = result->rows[j];
			for( i = 0; i < select->groupby->expr_count; i ++ ) {
				v1 = &row->groupby[i];
				x1 = select->groupby->expr[i];
				VAR_COMP_OP( v2, *v1, x1->var, ==, "==", 0, 0 );
				if( ! v2.iv )
					goto groupby_next;
			}
			goto eval_row;
groupby_next:
			continue;
		}
add_row:
		if( select->offset && rp < select->offset ) {
			rp ++;
			continue;
		}
		if( select->limit && rp - select->offset >= select->limit )
			goto finish;
		rp ++;
		Newxz( row, 1, csv_row_t );
		row->select = select;
		if( (result->row_count % ROW_COUNT_EXPAND) == 0 ) {
			Renew( result->rows, result->row_count + ROW_COUNT_EXPAND,
				csv_row_t * );
		}
		result->rows[result->row_count ++] = row;
		Newxz( row->data, result->column_count, csv_var_t );
		if( select->orderby != NULL )
			Newxz( row->orderby, select->orderby->expr_count, csv_var_t );
		if( select->groupby != NULL )
			Newxz( row->groupby, select->groupby->expr_count, csv_var_t );
		nr = 1;
eval_row:
		for( i = 0, j = 0; i < xl_cols->expr_count; i ++, j ++ ) {
			x1 = xl_cols->expr[i];
			if( x1->op == TK_TABLE ) {
				table2 = x1->table;
				for( l = 0; l < table2->column_count; l ++, j ++ ) {
					col = table2->columns + l;
					v1 = row->data + j;
					s1 = table->data + col->offset;
					l1 = col->length;
					if( l1 == 4 && strcmp( s1, "NULL" ) == 0 )
						continue;
					switch( col->type ) {
					case FIELD_TYPE_INTEGER:
						v1->iv = atoi( s1 );
						v1->flags = VAR_HAS_IV;
						break;
					case FIELD_TYPE_DOUBLE:
						if( table2->decimal_symbol != '.' ) {
							s2 = strchr( s1, table2->decimal_symbol );
							if( s2 != NULL )
								*s2 = '.';
						}
						v1->nv = atof( s1 );
						v1->flags = VAR_HAS_NV;
						break;
					case FIELD_TYPE_CHAR:
						v1->sv_len = charset_convert(
							s1, l1, table2->charset,
							csv->client_charset, &v1->sv
						);
						v1->flags = VAR_HAS_SV;
						break;
					case FIELD_TYPE_BLOB:
						if( l1 >= 2 && s1[0] == '0' && s1[1] == 'x' ) {
							l1 = (l1 - 2) / 2;
							if( ! l1 ) {
								v1->flags = 0;
								continue;
							}
							Renew( v1->sv, l1 + 1, char );
							for( m = 0, s1 += 2; m < l1; m ++ ) {
								ch = CHAR_FROM_HEX[*s1 & 0x7f] << 4;
								s1 ++;
								ch += CHAR_FROM_HEX[*s1 & 0x7f];
								s1 ++;
								v1->sv[m] = (char) ch;
							}
							v1->sv[l1] = '\0';
							v1->sv_len = l1;
							v1->flags = VAR_HAS_SV;
							break;
						}
					default:
						Renew( v1->sv, l1 + 1, char );
						Copy( s1, v1->sv, l1, char );
						v1->sv[l1] = '\0';
						v1->sv_len = l1;
						v1->flags = VAR_HAS_SV;
						break;
					}
				}
				continue;
			}
Ejemplo n.º 5
0
PerlIO_getpos(PerlIO *f, Fpos_t *pos)
{
 *pos = PerlIO_tell(f);
 return 0;
}
SV * parse_in_chunks(char * filepath, size_t filesize) {
    char *buf;
    size_t bytes_read = 0;
    int max_buf = 1000;
    char *err_msg;
    int block = BLOCK_HEADER;
    int cur_event_type = 0;
    int event_type = 0;
    char event_block = 0;
    char *brnl, *breq;
    AV * data;
    AV * datawrapper;
    AV * events;
    char *line;
    char * nl = "\n";
    char * eq = "=";
    int rewind_pos = 0;
    size_t cur_fpos = 0;
    SV * pbuf;
    SV * pmax_buf;

    AV * HANDLERS = get_av("Opsview::Utils::NDOLogsImporter::HANDLERS", 0);
    AV * INPUT_DATA_TYPE = get_av("Opsview::Utils::NDOLogsImporter::INPUT_DATA_TYPE", 0);

    int init_last_pos;
    int init_block;

    if ( first_read ) {
        if ( ! ( fh = PerlIO_open( filepath, "rb" ) ) ) {
            croak("Could not open file: %s\n", strerror(errno));
        }

        bytes_left = filesize;
        init_last_pos = prev_pos = first_read = 0;
        init_block = block = BLOCK_HEADER;
    } else {
        init_block = block = BLOCK_EVENTS;
        init_last_pos = prev_pos;
    }

    read_begin:


    brnl = NULL;
    breq = NULL;

    pbuf = get_sv("Opsview::Utils::NDOLogsImporter::PARSE_BUF", 0);
    pmax_buf = get_sv("Opsview::Utils::NDOLogsImporter::MAX_BUF_SIZE", 0);

    buf = SvPVX(pbuf);
    max_buf = SvIVX(pmax_buf);

    if ( max_buf < 1024 * 1024 && ! automated_tests ) {
        max_buf = 1024*1024;
        SvIV_set( pmax_buf, max_buf );
        SvGROW( pbuf, max_buf + 1);
        SvCUR_set( pbuf, max_buf);
    }

    if ( bytes_left > 0 ) {

        bytes_read = PerlIO_read(fh, buf + prev_pos, max_buf-prev_pos);
        cur_fpos = PerlIO_tell(fh);

        if ( bytes_read < 0 ) {
            err_msg = strerror(errno);

            PerlIO_close( fh );

            croak("Could not read file: %s\n", err_msg);
        }

        bytes_left -= bytes_read;

        events = (AV *)sv_2mortal((SV *)newAV());

        rewind_pos = last_999(buf+prev_pos, bytes_read);
        prev_pos = bytes_read + prev_pos - rewind_pos;
        buf[prev_pos] = '\0';

        // avg ratio events:file_size = 0.21%
        if ( prev_pos > 1000 ) {
            av_extend( events, (int)(prev_pos * 0.0021) );
        }


        for ( line = strtok_r(buf, nl, &brnl); line != NULL; line = strtok_r(NULL, nl, &brnl) )
        {
            switch(block) {
                case BLOCK_HEADER:
                    {
                        if ( strEQ(line, "STARTDATADUMP") ) {
                            block = BLOCK_EVENTS;
                        }
                    }
                    break;

                case BLOCK_EVENTS:
                    {
                        if ( strEQ(line, "1000") ) { /* NDO_API_ENDDATADUMP */
                            block = BLOCK_FOOTER;
                            continue;
                        }

                        cur_event_type = atoi(line);

                        /* ignore events we are not handling */
                        if ( !  av_exists(HANDLERS, cur_event_type) ) {
                            block = BLOCK_IGNORE_EVENT;
                            continue;
                        }

                        event_block = BLOCK_EVENT_STARTED;
                        if ( cur_event_type != event_type ) {
                            datawrapper = (AV *)sv_2mortal((SV *)newAV());
                            data = (AV *)sv_2mortal((SV *)newAV());

                            av_push( events, newSViv( cur_event_type ) );
                            av_push( datawrapper, newRV( (SV *)data ) );
                            av_push( events, newRV( (SV *)datawrapper ) );

                            event_type = cur_event_type;
                        } else {
                            data = (AV *)sv_2mortal((SV *)newAV());

                            av_push( datawrapper, newRV( (SV *)data ) );
                        }

                        block = BLOCK_EVENT; 
                    }
                    break;

                case BLOCK_EVENT:
                    {
                        if ( strEQ(line, "999") ) { /* NDO_API_ENDDATA */
                            block = BLOCK_EVENTS;
                            event_block = BLOCK_EVENT_ENDED;
                        } else {
                            char *k;
                            char *v;
                            int key;
                            int key_type = 0;
                            int v_len = 0;

                            k = strtok_r(line, eq, &breq); 
                            v = strtok_r(NULL, "\0", &breq);

                            key = atoi(k);
                            /* invalid key, skip parsing */
                            if ( key == 0 ) {
                                goto remove_invalid;
                            }

                            SV ** const k_type = av_fetch(INPUT_DATA_TYPE, key, 0 ); 
                            if ( k_type ) {
                                key_type = SvIVx( *k_type );
                            }

                            if ( v ) {
                                if ( key_type & 1 ) {
                                   v_len = ndo_unescape_buffer( v ); 
                                } else {
                                    v_len = strlen(v);
                                }
                            }

                            if ( key_type & 2 ) {
                                AV * datanstptr;
                                SV ** const datanst = av_fetch(data, key, 0 ); 
                                if ( datanst ) {
                                    datanstptr = (AV *)SvRV( *datanst );
                                } else {
                                    datanstptr = (AV *)sv_2mortal((SV *)newAV());

                                    av_store( data, key, newRV( (SV *)datanstptr ) );
                                }

                                if ( v ) { 
                                    av_push( datanstptr, newSVpvn(v, v_len) );
                                } else {
                                    av_push( datanstptr, newSVpvn("", 0) );
                                }

                            } else {
                                if ( v ) { 
                                    av_store( data, key, newSVpvn(v, v_len) );
                                } else {
                                    av_store( data, key, newSVpvn("", 0) );
                                }
                            }
                        }
                    }
                    break;

                case BLOCK_FOOTER:
                    {
                        if ( strEQ(line, "GOODBYE") ) {
                            block = BLOCK_HEADER;
                        }
                    }
                    break;

                case BLOCK_IGNORE_EVENT:
                    {
                        if ( strEQ(line, "999") ) { /* NDO_API_ENDDATA */
                            block = BLOCK_EVENTS; // go back to EVENTS
                            continue;
                        }
                    }
                    break;
            }
        };

        /* there were some events */
        if ( event_block != BLOCK_HEADER ) {
            if ( event_block != BLOCK_EVENT_ENDED ) {
                remove_invalid:
                    av_pop( datawrapper );
            }

            /* remove whole block if the last block has no events */
            if ( av_len( datawrapper ) == -1 ) {
                av_pop( events );
                av_pop( events );
            }
        }


        if ( av_len(events) > 0 ) {
            if ( rewind_pos > 0 && cur_fpos < filesize ) {
                memmove(buf, buf+prev_pos+1, rewind_pos-1);
            }

            prev_pos = rewind_pos - 1;

            return newRV_inc((SV *) events);
        } else {

            if ( cur_fpos < filesize && event_block != BLOCK_HEADER && event_block != BLOCK_EVENT_ENDED ) {
                int new_max_buf = max_buf * 2;

                SvIV_set( pmax_buf, new_max_buf );
                SvGROW( pbuf, new_max_buf + 1);
                SvCUR_set( pbuf, new_max_buf);
                //start again as previous buffer would be tokenized already
                prev_pos = 0;
                block = init_block;
                event_type = 0;


                PerlIO_close( fh );
                if ( ! ( fh = PerlIO_open( filepath, "rb" ) ) ) {
                    croak("Could not re-open file: %s\n", strerror(errno));
                }
                PerlIO_seek(fh, cur_fpos-bytes_read-init_last_pos, SEEK_SET);
                bytes_left += bytes_read + init_last_pos;

                goto read_begin; 
            }
        }
    }

    parser_reset_iterator();

    return &PL_sv_undef;
}
Ejemplo n.º 7
0
int
_check_buf(PerlIO *infile, Buffer *buf, int min_wanted, int max_wanted)
{
  int ret = 1;
  
  // Do we have enough data?
  if ( buffer_len(buf) < min_wanted ) {
    // Read more data
    uint32_t read;
    uint32_t actual_wanted;
    unsigned char *tmp;

#ifdef _MSC_VER
    uint32_t pos_check = PerlIO_tell(infile);
#endif
    
    if (min_wanted > max_wanted) {
      max_wanted = min_wanted;
    }
    
    // Adjust actual amount to read by the amount we already have in the buffer
    actual_wanted = max_wanted - buffer_len(buf);

    New(0, tmp, actual_wanted, unsigned char);
    
    DEBUG_TRACE("Buffering from file @ %d (min_wanted %d, max_wanted %d, adjusted to %d)\n",
      (int)PerlIO_tell(infile), min_wanted, max_wanted, actual_wanted
    );

    if ( (read = PerlIO_read(infile, tmp, actual_wanted)) <= 0 ) {
      if ( PerlIO_error(infile) ) {
#ifdef _MSC_VER
        // Show windows specific error message as Win32 PerlIO_read does not set errno
        DWORD last_error = GetLastError();
        LPWSTR *errmsg = NULL;
        FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, 0, last_error, 0, (LPWSTR)&errmsg, 0, NULL);
        warn("Error reading: %d %s (read %d wanted %d)\n", last_error, errmsg, read, actual_wanted);
        LocalFree(errmsg);
#else
        warn("Error reading: %s (wanted %d)\n", strerror(errno), actual_wanted);
#endif
      }
      else {
        warn("Error: Unable to read at least %d bytes from file.\n", min_wanted);
      }

      ret = 0;
      goto out;
    }

    buffer_append(buf, tmp, read);

    // Make sure we got enough
    if ( buffer_len(buf) < min_wanted ) {
      warn("Error: Unable to read at least %d bytes from file (only read %d).\n", min_wanted, read);
      ret = 0;
      goto out;
    }

#ifdef _MSC_VER
    // Bug 16095, weird off-by-one bug seen only on Win32 and only when reading a filehandle
    if (PerlIO_tell(infile) != pos_check + read) {
      //PerlIO_printf(PerlIO_stderr(), "Win32 bug, pos should be %d, but was %d\n", pos_check + read, PerlIO_tell(infile));
      PerlIO_seek(infile, pos_check + read, SEEK_SET);
    }
#endif

    DEBUG_TRACE("Buffered %d bytes, new pos %d\n", read, (int)PerlIO_tell(infile));

out:
    Safefree(tmp);
  }