예제 #1
0
파일: crawler.c 프로젝트: JDepooter/curl
int main(void)
{
  signal(SIGINT, sighandler);
  LIBXML_TEST_VERSION;
  curl_global_init(CURL_GLOBAL_DEFAULT);
  CURLM *multi_handle = curl_multi_init();
  curl_multi_setopt(multi_handle, CURLMOPT_MAX_TOTAL_CONNECTIONS, max_con);
  curl_multi_setopt(multi_handle, CURLMOPT_MAX_HOST_CONNECTIONS, 6L);

  /* enables http/2 if available */
#ifdef CURLPIPE_MULTIPLEX
  curl_multi_setopt(multi_handle, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
#endif

  /* sets html start page */
  curl_multi_add_handle(multi_handle, make_handle(start_page));

  int msgs_left;
  int pending = 0;
  int complete = 0;
  int still_running = 1;
  while(still_running && !pending_interrupt) {
    int numfds;
    curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
    curl_multi_perform(multi_handle, &still_running);

    /* See how the transfers went */
    CURLMsg *m = NULL;
    while((m = curl_multi_info_read(multi_handle, &msgs_left))) {
      if(m->msg == CURLMSG_DONE) {
        CURL *handle = m->easy_handle;
        char *url;
        memory *mem;
        curl_easy_getinfo(handle, CURLINFO_PRIVATE, &mem);
        curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &url);
        if(m->data.result == CURLE_OK) {
          long res_status;
          curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &res_status);
          if(res_status == 200) {
            char *ctype;
            curl_easy_getinfo(handle, CURLINFO_CONTENT_TYPE, &ctype);
            printf("[%d] HTTP 200 (%s): %s\n", complete, ctype, url);
            if(is_html(ctype) && mem->size > 100) {
              if(pending < max_requests && (complete + pending) < max_total) {
                pending += follow_links(multi_handle, mem, url);
                still_running = 1;
              }
            }
          }
          else {
            printf("[%d] HTTP %d: %s\n", complete, (int) res_status, url);
          }
        }
        else {
          printf("[%d] Connection failure: %s\n", complete, url);
        }
        curl_multi_remove_handle(multi_handle, handle);
        curl_easy_cleanup(handle);
        free(mem->buf);
        free(mem);
        complete++;
        pending--;
      }
    }
  }
  curl_multi_cleanup(multi_handle);
  curl_global_cleanup();
  return 0;
}
예제 #2
0
int main(int argc, char *argv[]) {
    char buf[PATH_MAX];
    char calling_link[PATH_MAX];
    char program_to_launch[PATH_MAX];
    char origin[PATH_MAX];
    char profile_bin_dir[PATH_MAX];
    char *s;
    int i;
    line = 0;

    s = getenv("HDIST_LAUNCHER_DEBUG");
    debug = (s != NULL && strcmp(s, "1") == 0);


    if (strstr(argv[0], "/") == NULL) {
        /* Launched through PATH lookup */
        if (find_in_path(argv[0], getenv("PATH"), calling_link, PATH_MAX) != 0) return -1;
    } else {
        hit_strlcpy(calling_link, argv[0], PATH_MAX);
    }


    /* Find calling link */
    if (debug) fprintf(stderr, "%sstart='%s'\n", debug_header, calling_link);
    if (follow_links(calling_link, profile_bin_dir, PATH_MAX) != 0) { line = __LINE__; goto error; }
    if (calling_link[0] == '\0') {
        help();
        return 0;
    }
    if (debug) fprintf(stderr, "%scaller=%s\n", debug_header, calling_link);
    if (debug) fprintf(stderr, "%sPROFILE_BIN_DIR=%s\n", debug_header, profile_bin_dir);

    if (profile_bin_dir[0] == '\0') {
        hit_strlcpy(profile_bin_dir, "__NA__", PATH_MAX);
    }

    /* Open ${calling_link}.link and read the contents */
    if (hit_strlcpy(buf, calling_link, PATH_MAX) >= PATH_MAX) { line = __LINE__; goto error; }
    if (hit_strlcat(buf, ".link", PATH_MAX) >= PATH_MAX) { line = __LINE__; goto error; }
    if (resolve_link_in_textfile(buf, program_to_launch, PATH_MAX) != 0) {
        if (errno != ENOENT) goto error;
        /* ${calling_link}.link not found, use ${calling_link}.real */
        if (hit_strlcpy(program_to_launch, calling_link, PATH_MAX) >= PATH_MAX) { line = __LINE__; goto error; }
        if (hit_strlcat(program_to_launch, ".real", PATH_MAX) >= PATH_MAX) { line = __LINE__; goto error; }
    }

    /* Find ${ORIGIN}; we need to resolve realpath() of program to launch */
    if (realpath(program_to_launch, origin) == NULL) {
        hit_strlcpy(origin, "__NA__", PATH_MAX);
    } else {
        splitpath(origin, &s);
        if (s == origin) {
            fprintf(stderr, "%sASSERTION FAILED, LINE %d\n", debug_header, __LINE__);
            return 127;
        }
    }
    
    if (debug) fprintf(stderr, "%sORIGIN=%s\n", debug_header, origin);

    if (debug) fprintf(stderr, "%sprogram=%s\n", debug_header, program_to_launch);
    if (attempt_shebang_launch(program_to_launch, origin, profile_bin_dir, argc, argv) == -1) {
        if (errno == ENOENT) {
            fprintf(stderr, "launcher:Unable to launch '%s' (%s)", program_to_launch, strerror(errno));
            return 127;
        }
        goto error;
    }
    /* shebang not present */
    /* substitute argv[0] with caller */
    argv[0] = calling_link;
    if (debug){
      fprintf(stderr, "%sprogram_to_launch=%s\n", debug_header, program_to_launch);
      for (i=0;i<argc;i++){
	fprintf(stderr, "%sargv[%d]=%s\n", debug_header, i,argv[i]);
      }
    }
    execv(program_to_launch, argv);
    fprintf(stderr, "launcher:Unable to launch '%s' (%s)\n", program_to_launch,
            strerror(errno));
    return 127;
    goto error;

 error:
    fprintf(stderr, "%s:%d:%s:\n", __FILE__, line, strerror(errno));
    return 127;

}