int main(int argc, const char * argv[]) { url_result_t url_r; // 结构体,主要包括all_url_list和existed_page, 用于存储爬取到的所有url。 urlq_t queue; // url队列 int num; // 用于向queue加入种子页面, queue中存储的是all_url_list中的位置 tpool_t *tpool; // 线程池 char *url_sed; // 种子网页 time_t starttime; // 起始时间 time_t endtime; // 结束时间 if (url_result_init(&url_r) == INIT_FAILURE) { printf("初始化url结果失败,退出程序\n"); exit(1); } if (queue_init(&queue) == INIT_FAILURE) { printf("初始化队列失败,退出程序\n"); exit(1); } if ((tpool = tpool_create(MAX_THREADS, &url_r, &queue)) == NULL) { printf("初始化线程池失败\n"); exit(1); } url_sed = "http://news.sohu.com/"; time(&starttime); printf("start work!\n"); // 休眠2秒,为了创造完所有的线程. sleep(2); pthread_mutex_lock(&tpool->mutex); // 首先把种子网页加入到线程中 num = url_result_add(&url_r, url_sed); queue_push(&queue, num); addStr(url_r.bloomfilter, url_sed); pthread_mutex_unlock(&tpool->mutex); pthread_cond_signal(&tpool->cond); while (queue.size > 0 || tpool->at_work != 0) { printf("queue_size: %d\n", queue.size); printf("front: %d\n", queue.front); printf("tail: %d\n", queue.tail); printf("list_size: %d\n", url_r.all_url_list_size); printf("at_work: %d\n", tpool->at_work); printf("existed_page_size: %d\n", url_r.existed_page_size); sleep(1); // 每隔1秒,输出一次日志。 } write_url_result_to_file(&url_r); tpool_destroy(); sleep(2); time(&endtime); printf("finish work"); printf("total time: %ld\n", endtime - starttime); printf("all_url_list_size: %d\n", url_r.all_url_list_size); printf("exited_page_size: %d\n", url_r.existed_page_size); return 0; }
static RIOTRtnE do_init(RServerCtx ** rtn_ctx) { RC_ASSERT(*rtn_ctx == NULL); RIOTRtnE ret; RServerCtx * ctx = NULL; if ((ctx = malloc(sizeof(RServerCtx))) == NULL) return RC_ERR_NOMEM; memset(ctx, 0, sizeof(RServerCtx)); ctx->status = RC_STATUS_INITIAL; ctx->cfg.port = RSERVER_CFG_DEFAULT_PORT; // Initial Evhr RC_LOG_D(" ==> [Evhr]"); if (evhr_create(&ctx->evhr) != EVHR_RTN_SUCCESS) { RC_LOG_E("Initial Evhr failed!"); ret = RC_ERR_NOMEM; goto error_return; } // Initial CusEv RC_LOG_D(" ==> [CusEv]"); if ((ctx->cusEv = evhr_cus_event_add( ctx->evhr, EVHR_PTR_TO_DATA(ctx), gRServerCusEvDispatch)) == NULL) { RC_LOG_E("Initial CusEv failed!"); ret = RC_ERR_NOMEM; goto error_return; } // Initial taskPool RC_LOG_D(" ==> [TaskPool]"); if (tpool_create(&ctx->taskPool, 5) != TPOOL_SUCCESS) { RC_LOG_E("Initial TaskPool failed!"); ret = RC_ERR_NOMEM; goto error_return; } // Initial RSUnit Socket RC_LOG_D(" ==> [RServerCSListener]"); if ((ret = RServerCSLtr_Init(&ctx->csLtr)) != RC_SUCCESS) { RC_LOG_E("Initial RServerCSListener failed! ret[%d]", ret); goto error_return; } *rtn_ctx = ctx; return RC_SUCCESS; error_return : do_destroy(ctx); return ret; }
int main(int argc, char **argv) { printf("--------- Server --------------\n"); int port = PORT; if (argc>1) port = atoi(argv[1]); //create thread pool if (tpool_create(THREAD_NUM) != 0) { printf("tpool_create failed\n"); exit(-1); } printf("--- Thread Pool Strat ---\n"); //initial server int listenfd = Server_init(port); socklen_t sockaddr_len = sizeof(struct sockaddr); //epoll static struct epoll_event ev, events[EPOLL_SIZE]; int epfd = epoll_create(EPOLL_SIZE); ev.events = EPOLLIN; ev.data.fd = listenfd; epoll_ctl(epfd, EPOLL_CTL_ADD, listenfd, &ev); while(1){ int events_count = epoll_wait(epfd, events, EPOLL_SIZE, -1); int i=0; //accept resquest for(; i<events_count; i++){ if(events[i].data.fd == listenfd) { int connfd; struct sockaddr_in clientaddr; while( ( connfd = accept(listenfd, (struct sockaddr *)&clientaddr, &sockaddr_len) ) > 0 ) { printf("EPOLL: Received New Connection Request---connfd= %d\n",connfd); struct args *p_args = (struct args *)malloc(sizeof(struct args)); p_args->fd = connfd; p_args->recv_finfo = recv_fileinfo; p_args->recv_fdata = recv_filedata; //add task to the work list tpool_add_work(worker, (void*)p_args); } } } } return 0; }
int main() { struct web_graph webg; int i; urlq_t queue; int num; tpool_t *tpool; if (init_webg(&webg) == INIT_FAIL) { printf("初始化图失败,退出程序!\n"); exit(1); } if (queue_init(&queue) == INIT_QUEUE_FAIL) { printf("初始化队列失败,退出程序!\n"); exit(1); } if ((tpool = tpool_create(MAX_THREADS, &webg, &queue)) == NULL) { printf("初始化线程池失败!\n"); exit(1); } printf("start work!\n"); //首先要将index.html加入点集和队列中 pthread_mutex_lock(&tpool->lock); num = insert_vertex(&webg, "/techqq/index.html"); queue_push(&queue, num); pthread_mutex_unlock(&tpool->lock); pthread_cond_signal(&tpool->cond); while (queue.size > 0 || tpool->at_work != 0) { printf("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@queue_size: %d@@@@@@@@@@@@@@@@@@@@@@@\n", queue.size); printf("front: %d\n", queue.front); printf("tail: %d\n", queue.tail); printf("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@list_size: %d@@@@@@@@@@@@@@@@@@@@@@@\n", webg.all_url_list_size); printf("$$$$$$$$$$$$$$$$$edge_set_size: %d\n", webg.edge_set_size); printf("at_work: %d\n", tpool->at_work); printf("---------------existed_page_size: %d\n", webg.existed_page_size); sleep(2); } printf("finish work!\n"); tpool_destroy(); printf("size: %d\n", webg.all_url_list_size); printf("queue_size: %d\n", queue.size); print_webg_to_file(&webg); destroy_webg(&webg); output_result_file(); }
int main(int argc,char **argv) { printf("Http server welcome you!\n"); tpool_create(10); if(1!=argc) getoption(argc,argv);//It's hard to learn how to use it if(NULL==_log) logfd=open(DEFAULTLOG,O_WRONLY | O_APPEND | O_CREAT); else logfd=open(_log,O_WRONLY | O_CREAT | O_APPEND); daytime(); int sockfd,sockfds; if(daemon_check) daemons(); ssl_init(ctx); signal(SIGPIPE,SIG_IGN); signal(SIGCHLD, SIG_IGN); sockfd=make_socket(sockfd); sockfds=make_socket_ssl(sockfds); if(sockfd<0||sockfds<0) errorfunc("sockfd error!"); int addrlen = 1; setsockopt(sockfd,SOL_SOCKET,SO_REUSEADDR,&addrlen,sizeof(addrlen));//set the port quickly reuse setsockopt(sockfds,SOL_SOCKET,SO_REUSEADDR,&addrlen,sizeof(addrlen)); struct epoll_event events[MAXEVENTS];//question int epollfd=epoll_create(MAXEVENTS); addfd(epollfd,sockfd,0); addfd(epollfd,sockfds,0); chdir("/home/wangyao/web"); while(1) { int ret=epoll_wait(epollfd,events,MAXEVENTS,-1);//epoll func should be use in here if(ret<0) errorfunc("epoll_wait error!"); lt(events,ret,epollfd,sockfd,sockfds); } close(sockfds); close(sockfd); close(epollfd); sleep(10); tpool_destroy(); SSL_CTX_free(ctx); exit(0); }
int main(int argc, char **argv) { printf("main start...........\n"); tpool_t *tpool = tpool_create(3); if(tpool == NULL) { printf("tpool_create failed...\n"); exit(1); } size_t i = 10; size_t a[10]; for(i = 0; i < 10; i++) { a[i] = i; tpool_task_add(tpool, func, (void*)&(a[i])); } tpool_destroy(tpool); printf("main end...........\n"); return 0; }
int ccnfdl_init(int pipeline_size) { int len; _listener.interest_pipe_size = pipeline_size; int pool_size = INTEREST_FLOWS; //int pool_size = 1; if (tpool_create(&_listener.interest_pipeline, pool_size) < 0) { log_critical(g_log, "tpool_create: could not create interest thread pool!"); return -1; } if ((_listener.sock = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) { log_critical(g_log, "socket: %s.", strerror(errno)); return -1; } memset(&(_listener.local), 0, sizeof(struct sockaddr_un)); _listener.local.sun_family = AF_UNIX; char sock_path[256]; ccnf_did2sockpath(g_nodeId, sock_path, 256); strcpy(_listener.local.sun_path, sock_path); unlink(_listener.local.sun_path); len = sizeof(struct sockaddr_un); if (bind(_listener.sock, (struct sockaddr * ) &(_listener.local), len) == -1) { log_critical(g_log, "bind: %s.", strerror(errno)); close(_listener.sock); return -1; } if (listen(_listener.sock, SOCK_QUEUE) == -1) { log_critical(g_log, "listen: %s.", strerror(errno)); return -1; } log_important(g_log, "interest window: %d", _listener.interest_pipe_size); log_important(g_log, "max interest window: %d", MAX_INTEREST_PIPELINE); return 0; }
/* * Given a list of directories to search, find all pools stored on disk. This * includes partial pools which are not available to import. If no args are * given (argc is 0), then the default directory (/dev/dsk) is searched. * poolname or guid (but not both) are provided by the caller when trying * to import a specific pool. */ static nvlist_t * zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t *iarg) { int i, dirs = iarg->paths; struct dirent64 *dp; char path[MAXPATHLEN]; char *end, **dir = iarg->path; size_t pathleft; nvlist_t *ret = NULL; static char *default_dir = "/dev/dsk"; pool_list_t pools = { 0 }; pool_entry_t *pe, *penext; vdev_entry_t *ve, *venext; config_entry_t *ce, *cenext; name_entry_t *ne, *nenext; avl_tree_t slice_cache; rdsk_node_t *slice; void *cookie; if (dirs == 0) { dirs = 1; dir = &default_dir; } /* * Go through and read the label configuration information from every * possible device, organizing the information according to pool GUID * and toplevel GUID. */ for (i = 0; i < dirs; i++) { tpool_t *t; char *rdsk; int dfd; boolean_t config_failed = B_FALSE; DIR *dirp; /* use realpath to normalize the path */ if (realpath(dir[i], path) == 0) { (void) zfs_error_fmt(hdl, EZFS_BADPATH, dgettext(TEXT_DOMAIN, "cannot open '%s'"), dir[i]); goto error; } end = &path[strlen(path)]; *end++ = '/'; *end = 0; pathleft = &path[sizeof (path)] - end; /* * Using raw devices instead of block devices when we're * reading the labels skips a bunch of slow operations during * close(2) processing, so we replace /dev/dsk with /dev/rdsk. */ if (strcmp(path, "/dev/dsk/") == 0) rdsk = "/dev/rdsk/"; else rdsk = path; if ((dfd = open64(rdsk, O_RDONLY)) < 0 || (dirp = fdopendir(dfd)) == NULL) { if (dfd >= 0) (void) close(dfd); zfs_error_aux(hdl, strerror(errno)); (void) zfs_error_fmt(hdl, EZFS_BADPATH, dgettext(TEXT_DOMAIN, "cannot open '%s'"), rdsk); goto error; } avl_create(&slice_cache, slice_cache_compare, sizeof (rdsk_node_t), offsetof(rdsk_node_t, rn_node)); /* * This is not MT-safe, but we have no MT consumers of libzfs */ while ((dp = readdir64(dirp)) != NULL) { const char *name = dp->d_name; if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0))) continue; slice = zfs_alloc(hdl, sizeof (rdsk_node_t)); slice->rn_name = zfs_strdup(hdl, name); slice->rn_avl = &slice_cache; slice->rn_dfd = dfd; slice->rn_hdl = hdl; slice->rn_nozpool = B_FALSE; avl_add(&slice_cache, slice); } /* * create a thread pool to do all of this in parallel; * rn_nozpool is not protected, so this is racy in that * multiple tasks could decide that the same slice can * not hold a zpool, which is benign. Also choose * double the number of processors; we hold a lot of * locks in the kernel, so going beyond this doesn't * buy us much. */ t = tpool_create(1, 2 * sysconf(_SC_NPROCESSORS_ONLN), 0, NULL); for (slice = avl_first(&slice_cache); slice; (slice = avl_walk(&slice_cache, slice, AVL_AFTER))) (void) tpool_dispatch(t, zpool_open_func, slice); tpool_wait(t); tpool_destroy(t); cookie = NULL; while ((slice = avl_destroy_nodes(&slice_cache, &cookie)) != NULL) { if (slice->rn_config != NULL && !config_failed) { nvlist_t *config = slice->rn_config; boolean_t matched = B_TRUE; if (iarg->poolname != NULL) { char *pname; matched = nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME, &pname) == 0 && strcmp(iarg->poolname, pname) == 0; } else if (iarg->guid != 0) { uint64_t this_guid; matched = nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &this_guid) == 0 && iarg->guid == this_guid; } if (!matched) { nvlist_free(config); } else { /* * use the non-raw path for the config */ (void) strlcpy(end, slice->rn_name, pathleft); if (add_config(hdl, &pools, path, config) != 0) config_failed = B_TRUE; } } free(slice->rn_name); free(slice); } avl_destroy(&slice_cache); (void) closedir(dirp); if (config_failed) goto error; } ret = get_configs(hdl, &pools, iarg->can_be_active); error: for (pe = pools.pools; pe != NULL; pe = penext) { penext = pe->pe_next; for (ve = pe->pe_vdevs; ve != NULL; ve = venext) { venext = ve->ve_next; for (ce = ve->ve_configs; ce != NULL; ce = cenext) { cenext = ce->ce_next; if (ce->ce_config) nvlist_free(ce->ce_config); free(ce); } free(ve); } free(pe); } for (ne = pools.names; ne != NULL; ne = nenext) { nenext = ne->ne_next; free(ne->ne_name); free(ne); } return (ret); }