static INDX_ROOT *_indx_parse(BD_FILE_H *fp)
{
BITSTREAM bs;
INDX_ROOT *index = calloc(1, sizeof(INDX_ROOT));
int indexes_start, extension_data_start;
if (!index) {
BD_DEBUG(DBG_CRIT, "out of memory\n");
return NULL;
}
bs_init(&bs, fp);
if (!_parse_header(&bs, &indexes_start, &extension_data_start) ||
!_parse_app_info(&bs, &index->app_info)) {
indx_free(&index);
return NULL;
}
bs_seek_byte(&bs, indexes_start);
if (!_parse_index(&bs, index)) {
indx_free(&index);
return NULL;
}
if (extension_data_start) {
BD_DEBUG(DBG_NAV | DBG_CRIT, "index.bdmv: unknown extension data at %d\n", extension_data_start);
}
return index;
}
//-----------------------------------------------------------------------------
int ContextSP::_sr_http_read_write()
{
if(_sock.set() & 1) {
if(0==_parse_header()){
return 0;
}
}
return _get_from_rock();
}
// Get the next valid MP3 frame.
bool mp3_get_frame(rbstream_p mFp, ssize_t *offset, uint32_t *fixed_header, void *buffer, uint32_t *size)
{
size_t frame_size;
for (;;) {
ssize_t n = _source_read_at(mFp, *offset, buffer, U32_LEN_IN_BYTES);
RETURN_VAL_IF_FAIL((n == U32_LEN_IN_BYTES), false);
uint32_t header = _u32_at((const uint8_t *)buffer);
if ((header & MP3_FRAME_HEADER_MASK) == (*fixed_header & MP3_FRAME_HEADER_MASK)
&& _parse_header(header, &frame_size)) {
break;
}
// Lost sync.
ssize_t pos = *offset;
if (!mp3_resync(mFp, *fixed_header, &pos, fixed_header)) {
// Unable to mp3_resync. Signalling end of stream.
return false;
}
*offset = pos;
// Policy: Pop out data when mCurrentPos updated!
rbs_seek_ext(mFp, *offset, SEEK_SET);
// Try again with the new position.
}
ssize_t n = _source_read_at(mFp, *offset, buffer, frame_size);
RETURN_VAL_IF_FAIL((n == (ssize_t) frame_size), false);
medvdbg("[%s] Line %d, pos %#x, framesize %#x\n", __FUNCTION__, __LINE__, *offset, frame_size);
*size = frame_size;
*offset += frame_size;
// Policy: Pop out data when mCurrentPos updated!
rbs_seek_ext(mFp, *offset, SEEK_SET);
return true;
}
static BDID_DATA *_bdid_parse(BD_FILE_H *fp)
{
BITSTREAM bs;
BDID_DATA *bdid = NULL;
uint32_t data_start, extension_data_start;
uint8_t tmp[16];
if (bs_init(&bs, fp) < 0) {
BD_DEBUG(DBG_NAV, "id.bdmv: read error\n");
return NULL;
}
if (!_parse_header(&bs, &data_start, &extension_data_start)) {
BD_DEBUG(DBG_NAV | DBG_CRIT, "id.bdmv: invalid header\n");
return NULL;
}
if (bs_seek_byte(&bs, 40) < 0) {
BD_DEBUG(DBG_NAV, "id.bdmv: read error\n");
return NULL;
}
bdid = calloc(1, sizeof(BDID_DATA));
if (!bdid) {
BD_DEBUG(DBG_CRIT, "out of memory\n");
return NULL;
}
bs_read_bytes(&bs, tmp, 4);
str_print_hex(bdid->org_id, tmp, 4);
bs_read_bytes(&bs, tmp, 16);
str_print_hex(bdid->disc_id, tmp, 16);
if (extension_data_start) {
BD_DEBUG(DBG_NAV | DBG_CRIT, "id.bdmv: ignoring unknown extension data\n");
}
return bdid;
}
static void _watch_parse(AccountFolder * folder, char const buf[], size_t read)
{
size_t i = 0;
while(i < read)
switch(folder->context)
{
case PC_FROM:
_parse_from(folder, buf, read, &i);
break;
case PC_GARBAGE:
_parse_garbage(folder, buf, read, &i);
break;
case PC_HEADER:
_parse_header(folder, buf, read, &i);
break;
case PC_BODY:
_parse_body(folder, buf, read, &i);
break;
}
folder->offset += read;
}
struct http_pack *http_parse(char *http_buf) {
struct http_pack *http;
struct fmap *header_map = NULL;
http = http_pack_create();
if (http == NULL) return NULL;
/* 1.parse start line */
if ((http_buf = _parse_request_line(http_buf, &http->start_line)) == NULL)
goto reject;
/* 2.parse headers */
if ((header_map = fmap_create_dupkey()) == NULL) return NULL;
header_map->type = &http_header_hash_type;
if ((http_buf = _parse_header(http_buf, header_map)) == NULL) goto reject;
http->headers = header_map;
/* 3.parse body with Content-Length
* TODO..to implement Transfer-Encoding may allow not Content-Length */
struct http_header_item *item = fmap_getvalue(http->headers, "Content-Length");
if (item == NULL) {
http->start_line.status = *_defined_status_get(411);
goto reject;
}
if (_parse_body(http_buf, (int) item->val_parse_func(item->val), http) == -1)
goto reject;
return http;
reject:
http->headers = NULL;
http->body = NULL;
fmap_free(header_map);
fstr_freestr(http->body);
return http;
}
static int _parse_line(char *f[], void **data, int len)
{
int i = atoi(f[F_RECTYPE]);
filetxt_job_rec_t **job = (filetxt_job_rec_t **)data;
filetxt_step_rec_t **step = (filetxt_step_rec_t **)data;
filetxt_header_t header;
_parse_header(f, &header);
switch(i) {
case JOB_START:
*job = _create_filetxt_job_rec(header);
(*job)->jobname = xstrdup(f[F_JOBNAME]);
(*job)->track_steps = atoi(f[F_TRACK_STEPS]);
(*job)->priority = atoi(f[F_PRIORITY]);
(*job)->ncpus = atoi(f[F_NCPUS]);
(*job)->nodes = xstrdup(f[F_NODES]);
for (i=0; (*job)->nodes[i]; i++) { /* discard trailing <CR> */
if (isspace((*job)->nodes[i]))
(*job)->nodes[i] = '\0';
}
if (!strcmp((*job)->nodes, "(null)")) {
xfree((*job)->nodes);
(*job)->nodes = xstrdup("(unknown)");
}
if (len > F_JOB_ACCOUNT) {
(*job)->account = xstrdup(f[F_JOB_ACCOUNT]);
for (i=0; (*job)->account[i]; i++) {
/* discard trailing <CR> */
if (isspace((*job)->account[i]))
(*job)->account[i] = '\0';
}
}
break;
case JOB_STEP:
*step = _create_filetxt_step_rec(header);
(*step)->stepnum = atoi(f[F_JOBSTEP]);
(*step)->status = atoi(f[F_STATUS]);
(*step)->exitcode = atoi(f[F_EXITCODE]);
(*step)->ntasks = atoi(f[F_NTASKS]);
(*step)->ncpus = atoi(f[F_STEPNCPUS]);
(*step)->elapsed = atoi(f[F_ELAPSED]);
(*step)->tot_cpu_sec = atoi(f[F_CPU_SEC]);
(*step)->tot_cpu_usec = atoi(f[F_CPU_USEC]);
(*step)->rusage.ru_utime.tv_sec = atoi(f[F_USER_SEC]);
(*step)->rusage.ru_utime.tv_usec = atoi(f[F_USER_USEC]);
(*step)->rusage.ru_stime.tv_sec = atoi(f[F_SYS_SEC]);
(*step)->rusage.ru_stime.tv_usec = atoi(f[F_SYS_USEC]);
(*step)->rusage.ru_maxrss = atoi(f[F_RSS]);
(*step)->rusage.ru_ixrss = atoi(f[F_IXRSS]);
(*step)->rusage.ru_idrss = atoi(f[F_IDRSS]);
(*step)->rusage.ru_isrss = atoi(f[F_ISRSS]);
(*step)->rusage.ru_minflt = atoi(f[F_MINFLT]);
(*step)->rusage.ru_majflt = atoi(f[F_MAJFLT]);
(*step)->rusage.ru_nswap = atoi(f[F_NSWAP]);
(*step)->rusage.ru_inblock = atoi(f[F_INBLOCKS]);
(*step)->rusage.ru_oublock = atoi(f[F_OUBLOCKS]);
(*step)->rusage.ru_msgsnd = atoi(f[F_MSGSND]);
(*step)->rusage.ru_msgrcv = atoi(f[F_MSGRCV]);
(*step)->rusage.ru_nsignals = atoi(f[F_NSIGNALS]);
(*step)->rusage.ru_nvcsw = atoi(f[F_NVCSW]);
(*step)->rusage.ru_nivcsw = atoi(f[F_NIVCSW]);
(*step)->stats.vsize_max = atoi(f[F_MAX_VSIZE]);
if (len > F_STEPNODES) {
(*step)->stats.vsize_max_taskid =
atoi(f[F_MAX_VSIZE_TASK]);
(*step)->stats.vsize_ave = atof(f[F_AVE_VSIZE]);
(*step)->stats.rss_max = atoi(f[F_MAX_RSS]);
(*step)->stats.rss_max_taskid =
atoi(f[F_MAX_RSS_TASK]);
(*step)->stats.rss_ave = atof(f[F_AVE_RSS]);
(*step)->stats.pages_max = atoi(f[F_MAX_PAGES]);
(*step)->stats.pages_max_taskid =
atoi(f[F_MAX_PAGES_TASK]);
(*step)->stats.pages_ave = atof(f[F_AVE_PAGES]);
(*step)->stats.cpu_min = atoi(f[F_MIN_CPU]);
(*step)->stats.cpu_min_taskid =
atoi(f[F_MIN_CPU_TASK]);
(*step)->stats.cpu_ave = atof(f[F_AVE_CPU]);
(*step)->stepname = xstrdup(f[F_STEPNAME]);
(*step)->nodes = xstrdup(f[F_STEPNODES]);
} else {
(*step)->stats.vsize_max_taskid = (uint16_t)NO_VAL;
(*step)->stats.vsize_ave = (float)NO_VAL;
(*step)->stats.rss_max = NO_VAL;
(*step)->stats.rss_max_taskid = (uint16_t)NO_VAL;
(*step)->stats.rss_ave = (float)NO_VAL;
(*step)->stats.pages_max = NO_VAL;
(*step)->stats.pages_max_taskid = (uint16_t)NO_VAL;
(*step)->stats.pages_ave = (float)NO_VAL;
(*step)->stats.cpu_min = NO_VAL;
(*step)->stats.cpu_min_taskid = (uint16_t)NO_VAL;
(*step)->stats.cpu_ave = (float)NO_VAL;
(*step)->stepname = NULL;
(*step)->nodes = NULL;
}
if (len > F_MIN_CPU_NODE) {
(*step)->stats.vsize_max_nodeid =
atoi(f[F_MAX_VSIZE_NODE]);
(*step)->stats.rss_max_nodeid =
atoi(f[F_MAX_RSS_NODE]);
(*step)->stats.pages_max_nodeid =
atoi(f[F_MAX_PAGES_NODE]);
(*step)->stats.cpu_min_nodeid =
atoi(f[F_MIN_CPU_NODE]);
} else {
(*step)->stats.vsize_max_nodeid = NO_VAL;
(*step)->stats.rss_max_nodeid = NO_VAL;
(*step)->stats.pages_max_nodeid = NO_VAL;
(*step)->stats.cpu_min_nodeid = NO_VAL;
}
if (len > F_STEP_ACCOUNT)
(*step)->account = xstrdup(f[F_STEP_ACCOUNT]);
if (len > F_STEP_REQUID)
(*step)->requid = atoi(f[F_STEP_REQUID]);
break;
case JOB_SUSPEND:
case JOB_TERMINATED:
*job = _create_filetxt_job_rec(header);
(*job)->elapsed = atoi(f[F_TOT_ELAPSED]);
(*job)->status = atoi(f[F_STATUS]);
if (len > F_JOB_REQUID)
(*job)->requid = atoi(f[F_JOB_REQUID]);
if (len > F_JOB_EXITCODE)
(*job)->exitcode = atoi(f[F_JOB_EXITCODE]);
break;
default:
error("UNKNOWN TYPE %d",i);
break;
}
return SLURM_SUCCESS;
}
srl_splitter_t * srl_build_splitter_struct(pTHX_ HV *opt) {
srl_splitter_t *splitter;
SV **svp;
STRLEN input_len;
splitter = srl_empty_splitter_struct(aTHX);
/* load options */
svp = hv_fetchs(opt, "input", 0);
if (svp && SvOK(*svp)) {
splitter->input_str = SvPV(*svp, input_len);
splitter->pos = splitter->input_str;
splitter->input_len = input_len;
splitter->input_str_end = splitter->input_str + input_len;
splitter->input_sv = SvREFCNT_inc(*svp);
SRL_SPLITTER_TRACE("input_size %" UVuf, input_len);
} else {
croak ("no input given");
}
svp = hv_fetchs(opt, "chunk_size", 0);
if (svp && SvOK(*svp)) {
splitter->size_limit = SvUV(*svp);
SRL_SPLITTER_TRACE("size_limit %" UVuf, splitter->size_limit);
}
splitter->compression_format = 0;
svp = hv_fetchs(opt, "compress", 0);
if (svp && SvOK(*svp)) {
IV compression_format = SvIV(*svp);
/* See also Splitter.pm's constants */
switch (compression_format) {
case 0:
SRL_SPLITTER_TRACE("no compression %s", "");
break;
case 1:
croak("incremental snappy compression not yet supported. Try gzip");
break;
case 2:
splitter->compression_format = 2;
SRL_SPLITTER_TRACE("gzip compression %s", "");
break;
default:
croak("invalid valie for 'compress' parameter");
}
}
splitter->header_str = NULL;
splitter->header_sv = NULL;
splitter->header_len = 0;
svp = hv_fetchs(opt, "header_data_template", 0);
if (svp && SvOK(*svp)) {
STRLEN header_len;
splitter->header_str = SvPV(*svp, header_len);
splitter->header_sv = SvREFCNT_inc(*svp);
splitter->header_len = header_len;
SRL_SPLITTER_TRACE("header_data_template found, of length %lu", header_len);
}
splitter->header_count_idx = -1;
svp = hv_fetchs(opt, "header_count_idx", 0);
if (svp && SvOK(*svp)) {
splitter->header_count_idx = SvIV(*svp);
SRL_SPLITTER_TRACE("header_count_idx found, %ld", splitter->header_count_idx);
}
_parse_header(aTHX_ splitter);
/* initialize stacks */
srl_splitter_stack_t * status_stack;
Newxz(status_stack, 1, srl_splitter_stack_t );
Newxz(status_stack->data, STACK_SIZE_INCR, UV );
status_stack->size = STACK_SIZE_INCR;
status_stack->top = 0;
splitter->status_stack = status_stack;
/* initialize */
splitter->deepness = 0;
char tag = *(splitter->pos);
splitter->pos++;
if (IS_SRL_HDR_ARRAYREF(tag)) {
int len = tag & 0xF;
splitter->input_nb_elts = len;
SRL_SPLITTER_TRACE(" * ARRAYREF of len, %d", len);
while (len-- > 0) {
stack_push(splitter->status_stack, ST_VALUE);
}
} else if (tag == SRL_HDR_REFN) {
tag = *(splitter->pos);
splitter->pos++;
if (tag == SRL_HDR_ARRAY) {
UV len = _read_varint_uv_nocheck(splitter);
splitter->input_nb_elts = len;
SRL_SPLITTER_TRACE(" * ARRAY of len, %lu", len);
while (len-- > 0) {
stack_push(splitter->status_stack, ST_VALUE);
}
} else {
croak("first tag is REFN but next tag is not ARRAY");
}
} else {
croak("first tag is not an ArrayRef");
}
/* now splitter->pos is on the first array element */
return splitter;
}
// Resync to next valid MP3 frame in the file.
static bool mp3_resync(rbstream_p fp, uint32_t match_header, ssize_t *inout_pos, uint32_t *out_header)
{
medvdbg("[%s] Line %d, match_header %#x, *pos %d\n", __FUNCTION__, __LINE__, match_header, *inout_pos);
if (*inout_pos == 0) {
// Skip an optional ID3 header if syncing at the very beginning of the datasource.
for (;;) {
uint8_t id3header[MP3_HEAD_ID3_TAG_LEN];
int retVal = _source_read_at(fp, *inout_pos, id3header, sizeof(id3header));
RETURN_VAL_IF_FAIL((retVal == (ssize_t) sizeof(id3header)), false);
if (memcmp("ID3", id3header, 3)) {
break;
}
// Skip the ID3v2 header.
size_t len = MP3_HEAD_ID3_FRAME_GETSIZE(id3header);
len += MP3_HEAD_ID3_TAG_LEN;
*inout_pos += len;
}
}
ssize_t pos = *inout_pos;
bool valid = false;
uint8_t buf[FRAME_RESYNC_READ_BYTES];
ssize_t bytesToRead = FRAME_RESYNC_READ_BYTES;
ssize_t totalBytesRead = 0;
ssize_t remainingBytes = 0;
bool reachEOS = false;
uint8_t *buf_ptr = buf;
do {
if (pos >= *inout_pos + FRAME_RESYNC_MAX_CHECK_BYTES) {
medvdbg("[%s] resync range < %d\n", __FUNCTION__, FRAME_RESYNC_MAX_CHECK_BYTES);
break;
}
if (remainingBytes < (ssize_t)U32_LEN_IN_BYTES) {
if (reachEOS) {
break;
}
memcpy(buf, buf_ptr, remainingBytes);
bytesToRead = FRAME_RESYNC_READ_BYTES - remainingBytes;
/*
* The next read position should start from the end of
* the last buffer, and thus should include the remaining
* bytes in the buffer.
*/
totalBytesRead = _source_read_at(fp, pos + remainingBytes, buf + remainingBytes, bytesToRead);
if (totalBytesRead <= 0) {
break;
}
reachEOS = (totalBytesRead != bytesToRead);
remainingBytes += totalBytesRead;
buf_ptr = buf;
continue;
}
uint32_t header = _u32_at(buf_ptr);
if (match_header != 0 && (header & MP3_FRAME_HEADER_MASK) != (match_header & MP3_FRAME_HEADER_MASK)) {
++pos;
++buf_ptr;
--remainingBytes;
continue;
}
size_t frame_size;
if (!_parse_header(header, &frame_size)) {
++pos;
++buf_ptr;
--remainingBytes;
continue;
}
// We found what looks like a valid frame,
// now find its successors.
valid = true;
ssize_t test_pos = pos + frame_size;
medvdbg("[%s] Line %d, valid frame at pos %#x + framesize %#x = %#x\n", __FUNCTION__, __LINE__, pos, frame_size, test_pos);
int j;
for (j = 0; j < FRAME_MATCH_REQUIRED; ++j) {
uint8_t temp[U32_LEN_IN_BYTES];
ssize_t retval = _source_read_at(fp, test_pos, temp, sizeof(temp));
if (retval < (ssize_t) sizeof(temp)) {
valid = false;
break;
}
uint32_t test_header = _u32_at(temp);
if ((test_header & MP3_FRAME_HEADER_MASK) != (header & MP3_FRAME_HEADER_MASK)) {
medvdbg("[%s] Line %d, invalid frame at pos1 %#x\n", __FUNCTION__, __LINE__, test_pos);
valid = false;
break;
}
size_t test_frame_size;
if (!_parse_header(test_header, &test_frame_size)) {
medvdbg("[%s] Line %d, invalid frame at pos2 %#x\n", __FUNCTION__, __LINE__, test_pos);
valid = false;
break;
}
medvdbg("[%s] Line %d, valid frame at pos %#x + framesize %#x = %#x\n", __FUNCTION__, __LINE__, test_pos, test_frame_size, test_pos + test_frame_size);
test_pos += test_frame_size;
}
if (valid) {
*inout_pos = pos;
if (out_header != NULL) {
*out_header = header;
}
medvdbg("[%s] Line %d, find header %#x at pos %d(%#x)\n", __FUNCTION__, __LINE__, header, pos, pos);
}
++pos;
++buf_ptr;
--remainingBytes;
} while (!valid);
return valid;
}
_mod_httpd_request_t *
_mod_httpd_request_parse_cmd (lw6sys_context_t * sys_context, _mod_httpd_reply_thread_data_t * reply_thread_data)
{
_mod_httpd_request_t *request = NULL;
lw6cnx_connection_t *cnx = reply_thread_data->cnx;
int eof = 0;
char *line = NULL;
lw6sys_log (sys_context, LW6SYS_LOG_DEBUG, _x_ ("process httpd cmd"));
request = (_mod_httpd_request_t *) LW6SYS_CALLOC (sys_context, sizeof (_mod_httpd_request_t));
if (request)
{
request->client_ip = lw6sys_str_copy (sys_context, cnx->remote_ip);
if (_mod_httpd_reply_thread_should_continue (sys_context, reply_thread_data))
{
request->first_line = lw6net_recv_line_tcp (sys_context, &(reply_thread_data->sock));
if (request->first_line)
{
if (_parse_first_line (sys_context, request))
{
if (lw6sys_str_is_null_or_empty (sys_context, cnx->password))
{
request->password_ok = 1;
}
while ((!eof) && _mod_httpd_reply_thread_should_continue (sys_context, reply_thread_data))
{
line = lw6net_recv_line_tcp (sys_context, &(reply_thread_data->sock));
if (line)
{
if (strlen (line) == 0)
{
eof = 1;
}
else
{
_parse_header (sys_context, request, line, cnx->local_url, cnx->password);
}
LW6SYS_FREE (sys_context, line);
}
}
}
}
}
if ((!request->first_line) || (request->get_head_post == 0) || (!request->uri))
{
/*
* OK, not parseable, we put dummy (empty) values in all fields to
* avoid suspicious NULL-string bugs, and let the following code
* return error 500.
*/
if (request->first_line)
{
LW6SYS_FREE (sys_context, request->first_line);
}
if (request->uri)
{
LW6SYS_FREE (sys_context, request->uri);
}
request->first_line = lw6sys_str_copy (sys_context, _ERROR_FIRST_LINE);
request->get_head_post = 0;
request->uri = lw6sys_str_copy (sys_context, _ERROR_URI);
}
}
return request;
}
_mod_httpd_request_t *
_mod_httpd_request_parse_oob (lw6sys_context_t * sys_context, _mod_httpd_context_t * httpd_context, lw6nod_info_t * node_info, lw6srv_oob_data_t * oob_data)
{
_mod_httpd_request_t *request = NULL;
int eof = 0;
char *line = NULL;
char b = '\0';
lw6sys_log (sys_context, LW6SYS_LOG_DEBUG, _x_ ("process httpd oob"));
/*
* If there's not even one byte available right now, it's no
* need to go further, we'll just return NULL and skip
* even the 500 error default trick, it's not even worth it.
*/
if (lw6net_tcp_peek (sys_context, &(oob_data->sock), &b, sizeof (b), 0) > 0)
{
request = (_mod_httpd_request_t *) LW6SYS_CALLOC (sys_context, sizeof (_mod_httpd_request_t));
if (request)
{
request->client_ip = lw6sys_str_copy (sys_context, oob_data->remote_ip);
if (_mod_httpd_oob_should_continue (sys_context, httpd_context, oob_data))
{
request->first_line = lw6net_recv_line_tcp (sys_context, &(oob_data->sock));
if (request->first_line)
{
if (_parse_first_line (sys_context, request))
{
if (lw6sys_str_is_null_or_empty (sys_context, node_info->const_info.password))
{
request->password_ok = 1;
}
while ((!eof) && _mod_httpd_oob_should_continue (sys_context, httpd_context, oob_data))
{
line = lw6net_recv_line_tcp (sys_context, &(oob_data->sock));
if (line)
{
if (strlen (line) == 0)
{
eof = 1;
}
else
{
_parse_header (sys_context, request, line, node_info->const_info.ref_info.url, node_info->const_info.password);
}
LW6SYS_FREE (sys_context, line);
}
}
}
}
}
if ((!request->first_line) || (request->get_head_post == 0) || (!request->uri))
{
/*
* Not parseable, user entered garbage but did enter something,
* we put dummy (empty) values in all fields to
* avoid suspicious NULL-string bugs, and let the following code
* return error 500.
*/
if (request->first_line)
{
LW6SYS_FREE (sys_context, request->first_line);
}
if (request->uri)
{
LW6SYS_FREE (sys_context, request->uri);
}
request->first_line = lw6sys_str_copy (sys_context, _ERROR_FIRST_LINE);
request->get_head_post = 0;
request->uri = lw6sys_str_copy (sys_context, _ERROR_URI);
}
}
}
return request;
}
