/*
General reader. Read a page and hand it over for processing.
*/
file_status_e
ogm_reader_c::read(generic_packetizer_c *,
bool) {
// Some tracks may contain huge gaps. We don't want to suck in the complete
// file.
if (get_queued_bytes() > 20 * 1024 * 1024)
return FILE_STATUS_HOLDING;
ogg_page og;
do {
// Make sure we have a page that we can work with.
if (read_page(&og) == FILE_STATUS_DONE)
return flush_packetizers();
// Is this the first page of a new stream? No, so process it normally.
if (!ogg_page_bos(&og))
process_page(&og);
} while (ogg_page_bos(&og));
size_t i;
// Are there streams that have not finished yet?
for (i = 0; i < sdemuxers.size(); i++)
if (!sdemuxers[i]->eos && sdemuxers[i]->in_use)
return FILE_STATUS_MOREDATA;
// No, we're done with this file.
return flush_packetizers();
}
/***********************************************************************
//this function will read the txt file line by line and store it into a vector
***********************************************************************/
void container::process_file(char const *argv[])
{
vector<string> lines;
int line_count = 0;
int page_count = 0;
string input;
ifstream file;
file.open(argv[1], ios_base::in);
//start reading if file opened successfully
if(file.is_open()) {
while(!file.eof()) {
getline(file, input,'\n');
if(file.eof()) {
break;
}
//insert line into vector
lines.push_back(input);
line_count++;
//process the page, clear lines vector,reset line_count, increment page_count
if(line_count == MAX_LINES) {
process_page(lines,page_count);
lines.clear();
line_count = 0;
page_count++;
}
}
file.close();
//process remaining lines, as a new page
if(!lines.empty())
process_page(lines, page_count);
}
}
/*=============================================================================
Function process_url_entry
Purpose: processes a URL entry for matches as specified in input args
Parameters:
*ue (IN) - a pointer to the url_entry to process
Returns: nothing, processes the url_entry
=============================================================================*/
void process_url_entry(struct url_entry *ue) {
int port;
char host[MAX_URL], path[MAX_URL], fullpath[MAX_URL], full_context[MAX_URL],
server_context[MAX_URL], buf[MAX_URL];
struct hostent *he;
struct sockaddr_in sad;
if (!parse_url(ue->url,host,&port,path,fullpath)) {
printf("error parsing URL: %s, URL discarded\n",ue->url);
return;
}
strcpy(full_context,"http://");
strncat(full_context, host, strlen(host));
if (port!=80) {
strncat(full_context,":",1);
sprintf(buf,"%d",port);
strncat(full_context,buf,strlen(buf));
}
strcpy(server_context,full_context);
strncat(full_context, path, strlen(path));
#ifdef DEBUG
printf("\nProcessing(%d) %s\n", ue->depth, ue->url);
printf(" host: %s\n",host);
printf(" port: %d\n",port);
printf(" path: %s\n",path);
printf(" fullpath: %s\n",fullpath);
printf(" full context: %s\n",full_context);
printf(" server context: %s\n",server_context);
#endif
if ((he=gethostbyname(host))==NULL) {
printf("unable to resolve host %s\n",host);
return;
}
memset((char *)&sad,0,sizeof(sad));
sad.sin_family = AF_INET;
sad.sin_port = htons(port);
memcpy(&sad.sin_addr,he->h_addr,he->h_length);
if (url_content_ok((struct sockaddr *)&sad,ue->url, host,port,fullpath))
process_page((struct sockaddr *)&sad,host,fullpath,ue->url,server_context,
full_context,ue->depth);
}
void process_file(PdfDocP doc, std::ostringstream& json) {
ptree doc_node;
doc_node.put("name", "document name");
int num_pages = doc->GetNumPages();
doc_node.put("num_pages", num_pages);
for (auto i = 0; i < num_pages; i++) {
ptree page_tree;
PdfPageP page = doc->AcquirePage(i);
if (!page)
continue;
process_page(page, page_tree);
doc->ReleasePage(page);
doc_node.add_child("page", page_tree);
}
ptree json_node;
json_node.add_child("document", doc_node);
write_json(json, json_node);
}
/*!
\brief handler_dispatch() calls handlers based on ImportParserFunc passed.
It passes off the pointer to the Vex_Import struct, the function arg,
the string read, and the pointer to the IOchannel representing the
input bytestream.
\param vex (Vex_Import *) The pointer to the Vex_Import datastructure.
\param function (ImportParserFunc) an enumeration used to determine
which handler to call.
\param arg (ImportParserArg) another enumeration passed to the functiosn
being dispatched from here
\param string (gchar *) The current line of the VEXfile just read. Used for
handlers than only need 1 lines's worth of data
\param iochannel (GIOChannel *) the pointer to the input stream of the
vexfile for reading additional data (used by some dispatched functions)
\see ImportParserFunc
\see ImportParserArg
\returns a GIOStatus of the dispatched function (usually G_IO_STATUS_NORMAL
or G_IO_STATUS_ERROR)
*/
G_MODULE_EXPORT GIOStatus handler_dispatch(Vex_Import *vex, ImportParserFunc function, ImportParserArg arg, gchar * string, GIOChannel *iochannel)
{
GIOStatus status = G_IO_STATUS_ERROR;
switch (function)
{
case VEX_HEADER:
status = process_header(vex, arg, string);
break;
case VEX_PAGE:
status = process_page(vex, string);
break;
case VEX_RANGE:
status = process_vex_range(vex, arg, string, iochannel);
break;
case VEX_TABLE:
status = process_vex_table(vex, string, iochannel);
break;
}
return status;
}
/*
* Scan the man section directory for pages and process each one,
* then check for junk in the corresponding cat section.
*/
static void
scan_section(char *mandir, char *section, char *cat_section)
{
struct dirent **entries;
char **expected = NULL;
int npages;
int nexpected = 0;
int i, e;
enum Ziptype zipped;
char *page_name;
char page_path[MAXPATHLEN];
char cat_path[MAXPATHLEN];
char zip_path[MAXPATHLEN];
/*
* scan the man section directory for pages
*/
npages = scandir(section, &entries, NULL, alphasort);
if (npages < 0) {
warn("%s/%s", mandir, section);
exit_code = 1;
return;
}
if (verbose || rm_junk) {
/*
* Maintain a list of all cat pages that should exist,
* corresponding to existing man pages.
*/
expected = (char **) calloc(npages, sizeof(char *));
}
for (i = 0; i < npages; free(entries[i++])) {
page_name = entries[i]->d_name;
snprintf(page_path, sizeof page_path, "%s/%s", section,
page_name);
if (!is_manpage_name(page_name)) {
if (!(test_path(page_path, NULL) & TEST_DIR)) {
junk(mandir, page_path,
"invalid man page name");
}
continue;
}
zipped = is_bzipped(page_name) ? BZIP :
is_gzipped(page_name) ? GZIP : NONE;
if (zipped != NONE) {
snprintf(cat_path, sizeof cat_path, "%s/%s",
cat_section, page_name);
if (expected != NULL)
expected[nexpected++] = strdup(page_name);
process_page(mandir, page_path, cat_path, zipped);
} else {
/*
* We've got an uncompressed man page,
* check to see if there's a (preferred)
* compressed one.
*/
snprintf(zip_path, sizeof zip_path, "%s%s",
page_path, GZ_EXT);
if (test_path(zip_path, NULL) != 0) {
junk(mandir, page_path,
"man page unused due to existing " GZ_EXT);
} else {
if (verbose) {
fprintf(stderr,
"warning, %s is uncompressed\n",
page_path);
}
snprintf(cat_path, sizeof cat_path, "%s/%s",
cat_section, page_name);
if (expected != NULL) {
asprintf(&expected[nexpected++],
"%s", page_name);
}
process_page(mandir, page_path, cat_path, NONE);
}
}
}
free(entries);
if (expected == NULL)
return;
/*
* scan cat sections for junk
*/
npages = scandir(cat_section, &entries, NULL, alphasort);
e = 0;
for (i = 0; i < npages; free(entries[i++])) {
const char *junk_reason;
int cmp = 1;
page_name = entries[i]->d_name;
if (strcmp(page_name, ".") == 0 || strcmp(page_name, "..") == 0)
continue;
/*
* Keep the index into the expected cat page list
* ahead of the name we've found.
*/
while (e < nexpected &&
(cmp = strcmp(page_name, expected[e])) > 0)
free(expected[e++]);
if (cmp == 0)
continue;
/* we have an unexpected page */
snprintf(cat_path, sizeof cat_path, "%s/%s", cat_section,
page_name);
if (!is_manpage_name(page_name)) {
if (test_path(cat_path, NULL) & TEST_DIR)
continue;
junk_reason = "invalid cat page name";
} else if (!is_gzipped(page_name) && e + 1 < nexpected &&
strncmp(page_name, expected[e + 1], strlen(page_name)) == 0 &&
strlen(expected[e + 1]) == strlen(page_name) + 3) {
junk_reason = "cat page unused due to existing " GZ_EXT;
} else
junk_reason = "cat page without man page";
junk(mandir, cat_path, junk_reason);
}
free(entries);
while (e < nexpected)
free(expected[e++]);
free(expected);
}
