int main(int argc, const char * argv[]) {
url_result_t url_r; // 结构体，主要包括all_url_list和existed_page, 用于存储爬取到的所有url。
urlq_t queue; // url队列
int num; // 用于向queue加入种子页面, queue中存储的是all_url_list中的位置
tpool_t *tpool; // 线程池
char *url_sed; // 种子网页
time_t starttime; // 起始时间
time_t endtime; // 结束时间
if (url_result_init(&url_r) == INIT_FAILURE) {
printf("初始化url结果失败，退出程序\n");
exit(1);
}
if (queue_init(&queue) == INIT_FAILURE) {
printf("初始化队列失败，退出程序\n");
exit(1);
}
if ((tpool = tpool_create(MAX_THREADS, &url_r, &queue)) == NULL) {
printf("初始化线程池失败\n");
exit(1);
}
url_sed = "http://news.sohu.com/";
time(&starttime);
printf("start work!\n");
// 休眠2秒，为了创造完所有的线程.
sleep(2);
pthread_mutex_lock(&tpool->mutex);
// 首先把种子网页加入到线程中
num = url_result_add(&url_r, url_sed);
queue_push(&queue, num);
addStr(url_r.bloomfilter, url_sed);
pthread_mutex_unlock(&tpool->mutex);
pthread_cond_signal(&tpool->cond);
while (queue.size > 0 || tpool->at_work != 0) {
printf("queue_size: %d\n", queue.size);
printf("front: %d\n", queue.front);
printf("tail: %d\n", queue.tail);
printf("list_size: %d\n", url_r.all_url_list_size);
printf("at_work: %d\n", tpool->at_work);
printf("existed_page_size: %d\n", url_r.existed_page_size);
sleep(1); // 每隔1秒，输出一次日志。
}
write_url_result_to_file(&url_r);
tpool_destroy();
sleep(2);
time(&endtime);
printf("finish work");
printf("total time: %ld\n", endtime - starttime);
printf("all_url_list_size: %d\n", url_r.all_url_list_size);
printf("exited_page_size: %d\n", url_r.existed_page_size);
return 0;
}
static RIOTRtnE do_init(RServerCtx ** rtn_ctx) {
RC_ASSERT(*rtn_ctx == NULL);
RIOTRtnE ret;
RServerCtx * ctx = NULL;
if ((ctx = malloc(sizeof(RServerCtx))) == NULL)
return RC_ERR_NOMEM;
memset(ctx, 0, sizeof(RServerCtx));
ctx->status = RC_STATUS_INITIAL;
ctx->cfg.port = RSERVER_CFG_DEFAULT_PORT;
// Initial Evhr
RC_LOG_D(" ==> [Evhr]");
if (evhr_create(&ctx->evhr) != EVHR_RTN_SUCCESS) {
RC_LOG_E("Initial Evhr failed!");
ret = RC_ERR_NOMEM;
goto error_return;
}
// Initial CusEv
RC_LOG_D(" ==> [CusEv]");
if ((ctx->cusEv = evhr_cus_event_add(
ctx->evhr, EVHR_PTR_TO_DATA(ctx),
gRServerCusEvDispatch)) == NULL) {
RC_LOG_E("Initial CusEv failed!");
ret = RC_ERR_NOMEM;
goto error_return;
}
// Initial taskPool
RC_LOG_D(" ==> [TaskPool]");
if (tpool_create(&ctx->taskPool, 5) != TPOOL_SUCCESS) {
RC_LOG_E("Initial TaskPool failed!");
ret = RC_ERR_NOMEM;
goto error_return;
}
// Initial RSUnit Socket
RC_LOG_D(" ==> [RServerCSListener]");
if ((ret = RServerCSLtr_Init(&ctx->csLtr)) != RC_SUCCESS) {
RC_LOG_E("Initial RServerCSListener failed! ret[%d]", ret);
goto error_return;
}
*rtn_ctx = ctx;
return RC_SUCCESS;
error_return :
do_destroy(ctx);
return ret;
}
int main(int argc, char **argv)
{
printf("--------- Server --------------\n");
int port = PORT;
if (argc>1)
port = atoi(argv[1]);
//create thread pool
if (tpool_create(THREAD_NUM) != 0) {
printf("tpool_create failed\n");
exit(-1);
}
printf("--- Thread Pool Strat ---\n");
//initial server
int listenfd = Server_init(port);
socklen_t sockaddr_len = sizeof(struct sockaddr);
//epoll
static struct epoll_event ev, events[EPOLL_SIZE];
int epfd = epoll_create(EPOLL_SIZE);
ev.events = EPOLLIN;
ev.data.fd = listenfd;
epoll_ctl(epfd, EPOLL_CTL_ADD, listenfd, &ev);
while(1){
int events_count = epoll_wait(epfd, events, EPOLL_SIZE, -1);
int i=0;
//accept resquest
for(; i<events_count; i++){
if(events[i].data.fd == listenfd)
{
int connfd;
struct sockaddr_in clientaddr;
while( ( connfd = accept(listenfd, (struct sockaddr *)&clientaddr, &sockaddr_len) ) > 0 )
{
printf("EPOLL: Received New Connection Request---connfd= %d\n",connfd);
struct args *p_args = (struct args *)malloc(sizeof(struct args));
p_args->fd = connfd;
p_args->recv_finfo = recv_fileinfo;
p_args->recv_fdata = recv_filedata;
//add task to the work list
tpool_add_work(worker, (void*)p_args);
}
}
}
}
return 0;
}
int main()
{
struct web_graph webg;
int i;
urlq_t queue;
int num;
tpool_t *tpool;
if (init_webg(&webg) == INIT_FAIL)
{
printf("初始化图失败，退出程序！\n");
exit(1);
}
if (queue_init(&queue) == INIT_QUEUE_FAIL)
{
printf("初始化队列失败，退出程序！\n");
exit(1);
}
if ((tpool = tpool_create(MAX_THREADS, &webg, &queue)) == NULL)
{
printf("初始化线程池失败！\n");
exit(1);
}
printf("start work!\n");
//首先要将index.html加入点集和队列中
pthread_mutex_lock(&tpool->lock);
num = insert_vertex(&webg, "/techqq/index.html");
queue_push(&queue, num);
pthread_mutex_unlock(&tpool->lock);
pthread_cond_signal(&tpool->cond);
while (queue.size > 0 || tpool->at_work != 0)
{
printf("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@queue_size: %d@@@@@@@@@@@@@@@@@@@@@@@\n", queue.size);
printf("front: %d\n", queue.front);
printf("tail: %d\n", queue.tail);
printf("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@list_size: %d@@@@@@@@@@@@@@@@@@@@@@@\n", webg.all_url_list_size);
printf("$$$$$$$$$$$$$$$$$edge_set_size: %d\n", webg.edge_set_size);
printf("at_work: %d\n", tpool->at_work);
printf("---------------existed_page_size: %d\n", webg.existed_page_size);
sleep(2);
}
printf("finish work!\n");
tpool_destroy();
printf("size: %d\n", webg.all_url_list_size);
printf("queue_size: %d\n", queue.size);
print_webg_to_file(&webg);
destroy_webg(&webg);
output_result_file();
}
int main(int argc,char **argv)
{
printf("Http server welcome you!\n");
tpool_create(10);
if(1!=argc)
getoption(argc,argv);//It's hard to learn how to use it
if(NULL==_log)
logfd=open(DEFAULTLOG,O_WRONLY | O_APPEND | O_CREAT);
else
logfd=open(_log,O_WRONLY | O_CREAT | O_APPEND);
daytime();
int sockfd,sockfds;
if(daemon_check)
daemons();
ssl_init(ctx);
signal(SIGPIPE,SIG_IGN);
signal(SIGCHLD, SIG_IGN);
sockfd=make_socket(sockfd);
sockfds=make_socket_ssl(sockfds);
if(sockfd<0||sockfds<0)
errorfunc("sockfd error!");
int addrlen = 1;
setsockopt(sockfd,SOL_SOCKET,SO_REUSEADDR,&addrlen,sizeof(addrlen));//set the port quickly reuse
setsockopt(sockfds,SOL_SOCKET,SO_REUSEADDR,&addrlen,sizeof(addrlen));
struct epoll_event events[MAXEVENTS];//question
int epollfd=epoll_create(MAXEVENTS);
addfd(epollfd,sockfd,0);
addfd(epollfd,sockfds,0);
chdir("/home/wangyao/web");
while(1)
{
int ret=epoll_wait(epollfd,events,MAXEVENTS,-1);//epoll func should be use in here
if(ret<0)
errorfunc("epoll_wait error!");
lt(events,ret,epollfd,sockfd,sockfds);
}
close(sockfds);
close(sockfd);
close(epollfd);
sleep(10);
tpool_destroy();
SSL_CTX_free(ctx);
exit(0);
}
int
main(int argc, char **argv)
{
printf("main start...........\n");
tpool_t *tpool = tpool_create(3);
if(tpool == NULL) {
printf("tpool_create failed...\n");
exit(1);
}
size_t i = 10;
size_t a[10];
for(i = 0; i < 10; i++) {
a[i] = i;
tpool_task_add(tpool, func, (void*)&(a[i]));
}
tpool_destroy(tpool);
printf("main end...........\n");
return 0;
}
int ccnfdl_init(int pipeline_size)
{
int len;
_listener.interest_pipe_size = pipeline_size;
int pool_size = INTEREST_FLOWS;
//int pool_size = 1;
if (tpool_create(&_listener.interest_pipeline, pool_size) < 0) {
log_critical(g_log, "tpool_create: could not create interest thread pool!");
return -1;
}
if ((_listener.sock = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
log_critical(g_log, "socket: %s.", strerror(errno));
return -1;
}
memset(&(_listener.local), 0, sizeof(struct sockaddr_un));
_listener.local.sun_family = AF_UNIX;
char sock_path[256];
ccnf_did2sockpath(g_nodeId, sock_path, 256);
strcpy(_listener.local.sun_path, sock_path);
unlink(_listener.local.sun_path);
len = sizeof(struct sockaddr_un);
if (bind(_listener.sock, (struct sockaddr * ) &(_listener.local), len) == -1) {
log_critical(g_log, "bind: %s.", strerror(errno));
close(_listener.sock);
return -1;
}
if (listen(_listener.sock, SOCK_QUEUE) == -1) {
log_critical(g_log, "listen: %s.", strerror(errno));
return -1;
}
log_important(g_log, "interest window: %d", _listener.interest_pipe_size);
log_important(g_log, "max interest window: %d", MAX_INTEREST_PIPELINE);
return 0;
}
/*
* Given a list of directories to search, find all pools stored on disk. This
* includes partial pools which are not available to import. If no args are
* given (argc is 0), then the default directory (/dev/dsk) is searched.
* poolname or guid (but not both) are provided by the caller when trying
* to import a specific pool.
*/
static nvlist_t *
zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t *iarg)
{
int i, dirs = iarg->paths;
struct dirent64 *dp;
char path[MAXPATHLEN];
char *end, **dir = iarg->path;
size_t pathleft;
nvlist_t *ret = NULL;
static char *default_dir = "/dev/dsk";
pool_list_t pools = { 0 };
pool_entry_t *pe, *penext;
vdev_entry_t *ve, *venext;
config_entry_t *ce, *cenext;
name_entry_t *ne, *nenext;
avl_tree_t slice_cache;
rdsk_node_t *slice;
void *cookie;
if (dirs == 0) {
dirs = 1;
dir = &default_dir;
}
/*
* Go through and read the label configuration information from every
* possible device, organizing the information according to pool GUID
* and toplevel GUID.
*/
for (i = 0; i < dirs; i++) {
tpool_t *t;
char *rdsk;
int dfd;
boolean_t config_failed = B_FALSE;
DIR *dirp;
/* use realpath to normalize the path */
if (realpath(dir[i], path) == 0) {
(void) zfs_error_fmt(hdl, EZFS_BADPATH,
dgettext(TEXT_DOMAIN, "cannot open '%s'"), dir[i]);
goto error;
}
end = &path[strlen(path)];
*end++ = '/';
*end = 0;
pathleft = &path[sizeof (path)] - end;
/*
* Using raw devices instead of block devices when we're
* reading the labels skips a bunch of slow operations during
* close(2) processing, so we replace /dev/dsk with /dev/rdsk.
*/
if (strcmp(path, "/dev/dsk/") == 0)
rdsk = "/dev/rdsk/";
else
rdsk = path;
if ((dfd = open64(rdsk, O_RDONLY)) < 0 ||
(dirp = fdopendir(dfd)) == NULL) {
if (dfd >= 0)
(void) close(dfd);
zfs_error_aux(hdl, strerror(errno));
(void) zfs_error_fmt(hdl, EZFS_BADPATH,
dgettext(TEXT_DOMAIN, "cannot open '%s'"),
rdsk);
goto error;
}
avl_create(&slice_cache, slice_cache_compare,
sizeof (rdsk_node_t), offsetof(rdsk_node_t, rn_node));
/*
* This is not MT-safe, but we have no MT consumers of libzfs
*/
while ((dp = readdir64(dirp)) != NULL) {
const char *name = dp->d_name;
if (name[0] == '.' &&
(name[1] == 0 || (name[1] == '.' && name[2] == 0)))
continue;
slice = zfs_alloc(hdl, sizeof (rdsk_node_t));
slice->rn_name = zfs_strdup(hdl, name);
slice->rn_avl = &slice_cache;
slice->rn_dfd = dfd;
slice->rn_hdl = hdl;
slice->rn_nozpool = B_FALSE;
avl_add(&slice_cache, slice);
}
/*
* create a thread pool to do all of this in parallel;
* rn_nozpool is not protected, so this is racy in that
* multiple tasks could decide that the same slice can
* not hold a zpool, which is benign. Also choose
* double the number of processors; we hold a lot of
* locks in the kernel, so going beyond this doesn't
* buy us much.
*/
t = tpool_create(1, 2 * sysconf(_SC_NPROCESSORS_ONLN),
0, NULL);
for (slice = avl_first(&slice_cache); slice;
(slice = avl_walk(&slice_cache, slice,
AVL_AFTER)))
(void) tpool_dispatch(t, zpool_open_func, slice);
tpool_wait(t);
tpool_destroy(t);
cookie = NULL;
while ((slice = avl_destroy_nodes(&slice_cache,
&cookie)) != NULL) {
if (slice->rn_config != NULL && !config_failed) {
nvlist_t *config = slice->rn_config;
boolean_t matched = B_TRUE;
if (iarg->poolname != NULL) {
char *pname;
matched = nvlist_lookup_string(config,
ZPOOL_CONFIG_POOL_NAME,
&pname) == 0 &&
strcmp(iarg->poolname, pname) == 0;
} else if (iarg->guid != 0) {
uint64_t this_guid;
matched = nvlist_lookup_uint64(config,
ZPOOL_CONFIG_POOL_GUID,
&this_guid) == 0 &&
iarg->guid == this_guid;
}
if (!matched) {
nvlist_free(config);
} else {
/*
* use the non-raw path for the config
*/
(void) strlcpy(end, slice->rn_name,
pathleft);
if (add_config(hdl, &pools, path,
config) != 0)
config_failed = B_TRUE;
}
}
free(slice->rn_name);
free(slice);
}
avl_destroy(&slice_cache);
(void) closedir(dirp);
if (config_failed)
goto error;
}
ret = get_configs(hdl, &pools, iarg->can_be_active);
error:
for (pe = pools.pools; pe != NULL; pe = penext) {
penext = pe->pe_next;
for (ve = pe->pe_vdevs; ve != NULL; ve = venext) {
venext = ve->ve_next;
for (ce = ve->ve_configs; ce != NULL; ce = cenext) {
cenext = ce->ce_next;
if (ce->ce_config)
nvlist_free(ce->ce_config);
free(ce);
}
free(ve);
}
free(pe);
}
for (ne = pools.names; ne != NULL; ne = nenext) {
nenext = ne->ne_next;
free(ne->ne_name);
free(ne);
}
return (ret);
}
