int main(int argc, char *argv[])
{
char *env;
int i, n, timeout = 250;
struct connection *conn;
method = "HEAD";
while ((i = getopt(argc, argv, "hp:t:vT:")) != -1)
switch ((char)i) {
case 'h':
usage(0);
case 'p':
set_proxy(optarg);
break;
case 't':
thread_limit = strtol(optarg, NULL, 0);
break;
case 'v':
verbose++;
break;
case 'T':
read_timeout = strtol(optarg, NULL, 0);
break;
default:
usage(1);
}
if (optind < argc)
while (optind < argc)
read_link_file(argv[optind++]);
else
read_urls(stdin);
/* set_proxy will not use this if proxy already set */
env = getenv("COMICS_PROXY");
if (env)
set_proxy(env);
if (thread_limit == 0) {
printf("You must allow at least one thread\n");
exit(1);
}
if (thread_limit > n_comics)
thread_limit = n_comics;
#ifdef _WIN32
win32_init();
#else
signal(SIGTERM, dump_outstanding);
signal(SIGHUP, dump_outstanding);
#endif
npoll = thread_limit + 1; /* add one for stdin */
ufds = must_calloc(npoll, sizeof(struct pollfd));
for (i = 0; i < npoll; ++i)
ufds[i].fd = -1;
while (head || outstanding) {
start_next_comic();
n = poll(ufds, npoll, timeout);
if (n < 0) {
my_perror("poll");
continue;
}
if (n == 0) {
timeout_connections();
if (!start_next_comic())
/* Once we have all the comics
* started, increase the timeout
* period. */
timeout = 1000;
continue;
}
for (conn = comics; conn; conn = conn->next)
if (!conn->poll)
continue;
else if (conn->poll->revents & POLLOUT) {
if (!conn->connected)
check_connect(conn);
else {
time(&conn->access);
write_request(conn);
}
} else if (conn->poll->revents & POLLIN) {
/* This check is needed for openssl */
if (!conn->connected)
check_connect(conn);
else
read_conn(conn);
}
}
out_results(comics, 0);
return n_comics != gotit;
}
int main(int argc, char *argv[])
{
char *env;
int i, verify = 0, clean = 0;
while ((i = getopt(argc, argv, "cd:hi:kl:p:t:vCT:V")) != -1)
switch ((char)i) {
case 'c':
clean = 1;
break;
case 'd':
comics_dir = must_strdup(optarg);
break;
case 'h':
usage(0);
case 'i':
add_index_dir(optarg);
break;
case 'k':
unlink_index = 0;
break;
case 'l':
links_only = fopen(optarg, "w");
if (!links_only) {
my_perror(optarg);
exit(1);
}
break;
case 'p':
set_proxy(optarg);
break;
case 't':
thread_limit = strtol(optarg, NULL, 0);
threads_set = 1;
break;
case 'v':
verbose++;
break;
case 'C':
#ifdef WANT_SSL
openssl_list_ciphers();
#else
puts("-C not supported.");
#endif
exit(0);
case 'T':
read_timeout = strtol(optarg, NULL, 0);
break;
case 'V':
verify = 1;
break;
default:
usage(1);
}
if (optind < argc)
while (optind < argc) {
if (read_config(argv[optind])) {
printf("Fatal error in config file\n");
exit(1);
}
++optind;
}
else if (read_config(NULL)) {
printf("Fatal error in config file\n");
exit(1);
}
if (verify) {
printf("Comics: %u Skipped today: %u\n", n_comics + skipped, skipped);
if (verbose)
dump_outstanding(0);
return 0;
}
/* set_proxy will not use this if proxy already set */
env = getenv("COMICS_PROXY");
if (env)
set_proxy(env);
if (thread_limit == 0) {
printf("You must allow at least one thread\n");
exit(1);
}
if (thread_limit > n_comics)
thread_limit = n_comics;
cd_comics_dir(clean);
#ifdef _WIN32
win32_init();
#else
signal(SIGTERM, dump_outstanding);
signal(SIGHUP, dump_outstanding);
signal(SIGPIPE, sigpipe);
#endif
if (links_only)
fclose(links_only);
want_extensions = 1;
main_loop();
out_results(comics, skipped);
#ifdef WIN32
printf("Hit return to exit");
getchar();
#endif
free_cache(); /* for valgrind */
free_comics(); /* for valgrind */
if (debug_fp)
fclose(debug_fp);
return 0;
}
void
do_intersect2(uint numPhrases, uint REPETITIONS)
{
uint iters,currPhrase;
int error = 0;
double time, timeE, tot_time = 0;
uint **patternSets; uint numIdsPerPattern; uint numOfPatterns;
readPatternSetsIntersect (&numIdsPerPattern, &numOfPatterns, &patternSets);
#ifdef DEBUG_ON
uint i,j;
fprintf(stderr,"\n parameters: maxNunPhr %d, numPhr %d, numIdPerPhr=%d",numPhrases , numOfPatterns,numIdsPerPattern);
fprintf(stderr,"\n patterns to intersect (1 list per row) = \n");
for (i=0;i<numOfPatterns; i++) {
for (j=0;j<numIdsPerPattern;j++)
fprintf(stderr,"[%d]", patternSets[i][j]);
fprintf(stderr,"\n");
}
fprintf(stderr,"\n");
#endif
if (numPhrases > numOfPatterns) {
fprintf(stderr,"\nBuild_il: Error during intersect2 %u/%u IDs\n", numPhrases, numOfPatterns);
exit(0);
}
//numPhrases = numOfPatterns;
#ifdef DO_LOG
static char log_file_name[10]="log.txt";
init_log(log_file_name);
#endif
uint *occs; uint numocc=0; uint tot_numocc=0;
iters=0;
currPhrase=0;
for (currPhrase=0; currPhrase<numPhrases; currPhrase++) {
#ifdef DEBUG_ON
fprintf(stderr,"\n intersect Phrase-Pattern <<%u>> -->", currPhrase);
for (i=0;i<numIdsPerPattern;i++) fprintf(stderr,"[%d]", patternSets[currPhrase][i]);
#endif
/* extract */
time = getTime ();
for (iters=0;iters<REPETITIONS;iters++) {
if (iters) free (occs); //not in the first one.
error = intersect_2_il (Index, patternSets[currPhrase][0], patternSets[currPhrase][1],
&numocc, &occs);
}
timeE=(getTime () - time)/REPETITIONS;
tot_time += timeE;
tot_numocc += numocc;
// fprintf(stderr,"\n i %u, occs = %u",currPhrase,numocc);
#ifdef DO_LOG
out_results(patternSets[currPhrase], numIdsPerPattern, occs, numocc);
#endif
plotResults(patternSets[currPhrase][0], patternSets[currPhrase][1], timeE, currPhrase+1, matlab, outfilename);
// uint i;
// fprintf(stderr,"\n ******** %d Ocurrencias found ...",numocc);
// for ( i=0;i<numocc; i++) {fprintf(stderr,"\n occ[%3d] ==> %d",i,occs[i]);}
IFERRORIL (error);
if (Verbose) {
fwrite(&(currPhrase), sizeof(uint), 1, stdout);
fwrite(&numocc, sizeof(numocc), 1, stdout);
}
#ifdef DEBUG_ON
fprintf(stderr,"\n NUMOCCS = %d ",numocc);fflush(stderr);
#endif
//fprintf(stderr,"\n NUMOCCS = %d ",numocc);fflush(stderr);
if (numocc)
free (occs); //the resulting list has always allocated memory even though numoccs==0;
numocc=0;
}
#ifdef DO_LOG
end_log() ;
#endif
endResults( matlab, outfilename, numPhrases);
freePatternSetsIntersect(patternSets, numOfPatterns);
fprintf (stderr, "\nTotal Num occs found = %lu\n", (ulong)tot_numocc);
fprintf (stderr, "Int2 time = %.8f msecs\n", tot_time*1);
fprintf (stderr, "Int2_time/numOfPatterns = %.8f msecs/patt\n\n",
(tot_time * 1) / numOfPatterns);
fprintf (stderr, "(Load_time+Int2_time)/numOfPatterns = %.8f msecs/patt\n\n",
((Load_time+tot_time) * 1) / numOfPatterns);
fprintf (stderr, "Int2_time/occ = %.8f msecs per post \n\n",
(tot_time * 1) / tot_numocc);
/**/ fprintf(timeFile,"\n\t occs = %9lu :: time = %.8f msecs ",(ulong)tot_numocc,tot_time*1);
/**/ fprintf(timeFile,"\n\t avg time per pattern = %.8f msecs ",(tot_time * 1) / numOfPatterns);
/**/ fprintf(timeFile,"\n\t avg time per occur = %.8f msecs \n",(tot_time * 1) / tot_numocc);
}
void do_intersectN(uint numPhrases, uint REPETITIONS){
uint iters,currPhrase;
int error = 0;
double time, tot_time = 0;
uint **patternSets; uint numIdsPerPattern; uint numOfPatterns;
readPatternSetsIntersect (&numIdsPerPattern, &numOfPatterns, &patternSets);
#ifdef DEBUG_ON
uint i,j;
fprintf(stderr,"\n parameters: maxNunPhr %d, numPhr %d, numIdPerPhr=%d",numPhrases , numOfPatterns,numIdsPerPattern);
fprintf(stderr,"\n patterns to intersect (1 list per row) = \n");
for (i=0;i<numOfPatterns; i++) {
for (j=0;j<numIdsPerPattern;j++)
fprintf(stderr,"[%d]", patternSets[i][j]);
fprintf(stderr,"\n");
}
fprintf(stderr,"\n");
#endif
if (numPhrases > numOfPatterns) {
fprintf(stderr,"\nBuild_il: Error during intersect2 %u/%u IDs\n", numPhrases, numOfPatterns);
exit(0);
}
//numPhrases = numOfPatterns;
#ifdef DO_LOG
static char log_file_name[10]="logN.txt";
init_log(log_file_name);
#endif
uint *occs; uint numocc=0; uint tot_numocc=0;
iters=0;
currPhrase=0;
for (currPhrase=0; currPhrase<numPhrases; currPhrase++) {
#ifdef DEBUG_ON
fprintf(stderr,"\n intersect Phrase-Pattern <<%u>> \n", currPhrase);
#endif
/* extract */
time = getTime ();
for (iters=0;iters<REPETITIONS;iters++) {
if (numocc) free (occs);
error = intersect_N_il (Index, patternSets[currPhrase], numIdsPerPattern,
&numocc, &occs);
}
tot_time += (getTime () - time)/REPETITIONS;
tot_numocc += numocc;
#ifdef DO_LOG
out_results(patternSets[currPhrase], numIdsPerPattern, occs, numocc);
#endif
IFERRORIL (error);
if (Verbose) {
fwrite(&(currPhrase), sizeof(uint), 1, stdout);
fwrite(&numocc, sizeof(numocc), 1, stdout);
}
#ifdef DEBUG_ON
fprintf(stderr,"\n NUMOCCS = %d ",numocc);fflush(stderr);
#endif
if (numocc) free (occs);
numocc=0;
}
#ifdef DO_LOG
end_log() ;
#endif
freePatternSetsIntersect(patternSets, numOfPatterns);
fprintf (stderr, "Total Num occs found = %lu\n", (ulong)tot_numocc);
fprintf (stderr, "IntN time = %.8f msecs\n", tot_time*1);
fprintf (stderr, "IntN_time/numOfPatterns = %.8f msecs/patt\n\n",
(tot_time * 1) / numOfPatterns);
fprintf (stderr, "(Load_time+IntN_time)/numOfPatterns = %.8f msecs/patt\n\n",
((Load_time+tot_time) * 1) / numOfPatterns);
fprintf (stderr, "IntN_time/occ = %.8f msecs per post \n\n",
(tot_time * 1) / tot_numocc);
/**/ fprintf(timeFile,"\n\t occs = %9lu :: time = %.8f msecs ",(ulong)tot_numocc,tot_time*1);
/**/ fprintf(timeFile,"\n\t avg time per pattern = %.8f msecs ",(tot_time * 1) / numOfPatterns);
/**/ fprintf(timeFile,"\n\t avg time per occur = %.8f msecs \n",(tot_time * 1) / tot_numocc);
}
void
do_extract(uint numIdsToExtract, uint REPETITIONS)
{
uint iters,currID;
int error = 0;
double time, tot_time = 0;
uint *IDs; uint numOfIDs;
readPatternSetsExtract (&numOfIDs, &IDs); //max number of ids to extract
#ifdef DEBUG_ON
uint i;
fprintf(stderr,"\n IDS = ");
for (i=0;i<numOfIDs; i++) {fprintf(stderr,"[%d]", IDs[i]);}
fprintf(stderr,"\n");
#endif
if (numIdsToExtract > numOfIDs) {
fprintf(stderr,"\nBuild_il: Error during extract %u/%u IDs\n", numIdsToExtract, numOfIDs);
exit(0);
}
numOfIDs = numIdsToExtract;
uint *occs; uint numocc=0; uint tot_numocc=0;
iters=0;
currID=0;
for (currID=0; currID<numOfIDs; currID++) {
#ifdef DEBUG_ON
fprintf(stderr,"\n to extract List of ID <<%u>> \n", IDs[currID]);
#endif
/* extract */
time = getTime ();
#ifdef DO_LOG
static char log_file_name[10]="logE.txt";
init_log(log_file_name);
#endif
for (iters=0;iters<REPETITIONS;iters++) {
if (numocc) free (occs);
error = extractList_il (Index, IDs[currID], &occs, &numocc);
}
tot_time += (getTime () - time)/REPETITIONS;
tot_numocc += numocc;
#ifdef DO_LOG
out_results(&(IDs[currID]), 1, occs, numocc);
#endif
IFERRORIL (error);
if (Verbose) {
fwrite(&(IDs[currID]), sizeof(uint), 1, stdout);
fwrite(&numocc, sizeof(numocc), 1, stdout);
}
if (numocc) free (occs);
numocc=0;
}
#ifdef DO_LOG
end_log() ;
#endif
freePatternSetsExtract(IDs);
fprintf (stderr, "Total Num occs found = %lu\n", (ulong)tot_numocc);
fprintf (stderr, "extract time = %.8f msecs\n", tot_time*1);
fprintf (stderr, "extract_time/Num_IDs = %.8f msecs/ID\n\n",
(tot_time * 1) / numOfIDs);
fprintf (stderr, "(Load_time+extract_time)/Num_IDs = %.8f msecs/ID\n\n",
((Load_time+tot_time) * 1) / numOfIDs);
fprintf (stderr, "extract_time/occ = %.8f msecs per post \n\n",
(tot_time * 1) / tot_numocc);
/**/ fprintf(timeFile,"\n\t occs = %9lu :: time = %.8f msecs ",(ulong)tot_numocc,tot_time*1);
/**/ fprintf(timeFile,"\n\t avg time per pattern = %.8f msecs ",(tot_time * 1) / numOfIDs);
/**/ fprintf(timeFile,"\n\t avg time per occur = %.8f msecs \n",(tot_time * 1) / tot_numocc);
}
