/**
* Send data to ReactorThread
*/
int swWorker_send2reactor(swEventData *ev_data, size_t sendn, int fd)
{
int ret;
swServer *serv = SwooleG.serv;
/**
* reactor_id: The fd in which the reactor.
*/
int reactor_id = ev_data->info.from_id;
int pipe_index = fd % serv->reactor_pipe_num;
/**
* pipe_worker_id: The pipe in which worker.
*/
int pipe_worker_id = reactor_id + (pipe_index * serv->reactor_num);
swWorker *worker = swServer_get_worker(serv, pipe_worker_id);
if (SwooleG.main_reactor)
{
ret = SwooleG.main_reactor->write(SwooleG.main_reactor, worker->pipe_worker, ev_data, sendn);
}
else
{
ret = swSocket_write_blocking(worker->pipe_worker, ev_data, sendn);
}
return ret;
}
static int swFactoryProcess_start(swFactory *factory)
{
int i;
swServer *serv = factory->ptr;
swWorker *worker;
for (i = 0; i < serv->worker_num; i++)
{
worker = swServer_get_worker(serv, i);
if (swWorker_create(worker) < 0)
{
return SW_ERR;
}
}
serv->reactor_pipe_num = serv->worker_num / serv->reactor_num;
//必须先启动manager进程组，否则会带线程fork
if (swManager_start(factory) < 0)
{
swWarn("swFactoryProcess_manager_start failed.");
return SW_ERR;
}
//主进程需要设置为直写模式
factory->finish = swFactory_finish;
return SW_OK;
}
void swWorker_onStop(swServer *serv)
{
if (serv->onWorkerStop)
{
serv->onWorkerStop(serv, SwooleWG.id);
}
swWorker_free(swServer_get_worker(serv, SwooleWG.id));
}
int swFactoryProcess_start(swFactory *factory)
{
if (swFactory_check_callback(factory) < 0)
{
swWarn("swFactory_check_callback fail");
return SW_ERR;
}
swServer *serv = factory->ptr;
swFactoryProcess *object = factory->object;
object->workers_status = SwooleG.memory_pool->alloc(SwooleG.memory_pool, sizeof(char)*serv->worker_num);
//worler idle or busy
if (object->workers_status == NULL)
{
swWarn("alloc for worker_status fail");
return SW_ERR;
}
//保存下指针，需要和reactor做绑定
serv->workers = object->workers;
int i;
swWorker *worker;
for(i = 0; i < serv->worker_num; i++)
{
worker = swServer_get_worker(serv, i);
if (swWorker_create(worker) < 0)
{
return SW_ERR;
}
}
//必须先启动manager进程组，否则会带线程fork
if (swFactoryProcess_manager_start(factory) < 0)
{
swWarn("swFactoryProcess_manager_start fail");
return SW_ERR;
}
if (serv->ipc_mode == SW_IPC_MSGQUEUE)
{
//tcp & message queue require writer pthread
if (serv->have_tcp_sock == 1)
{
int ret = swFactoryProcess_writer_start(factory);
if (ret < 0)
{
return SW_ERR;
}
}
}
//主进程需要设置为直写模式
factory->finish = swFactory_finish;
return SW_OK;
}
/**
* worker main loop
*/
int swWorker_loop(swFactory *factory, int worker_id)
{
swServer *serv = factory->ptr;
#ifndef SW_WORKER_USE_SIGNALFD
SwooleG.use_signalfd = 0;
#endif
//worker_id
SwooleWG.id = worker_id;
SwooleWG.request_count = 0;
SwooleG.pid = getpid();
//signal init
swWorker_signal_init();
swWorker *worker = swServer_get_worker(serv, worker_id);
swServer_worker_init(serv, worker);
SwooleG.main_reactor = sw_malloc(sizeof(swReactor));
if (SwooleG.main_reactor == NULL)
{
swError("[Worker] malloc for reactor failed.");
return SW_ERR;
}
if (swReactor_create(SwooleG.main_reactor, SW_REACTOR_MAXEVENTS) < 0)
{
swError("[Worker] create worker_reactor failed.");
return SW_ERR;
}
serv->workers[worker_id].status = SW_WORKER_IDLE;
int pipe_worker = serv->workers[worker_id].pipe_worker;
swSetNonBlock(pipe_worker);
SwooleG.main_reactor->ptr = serv;
//这里的add函数是epoll的add函数
SwooleG.main_reactor->add(SwooleG.main_reactor, pipe_worker, SW_FD_PIPE | SW_EVENT_READ);
SwooleG.main_reactor->setHandle(SwooleG.main_reactor, SW_FD_PIPE, swWorker_onPipeReceive);
SwooleG.main_reactor->setHandle(SwooleG.main_reactor, SW_FD_PIPE | SW_FD_WRITE, swReactor_onWrite);
swWorker_onStart(serv);
#ifdef HAVE_SIGNALFD
if (SwooleG.use_signalfd)
{
swSignalfd_setup(SwooleG.main_reactor);
}
#endif
//main loop
SwooleG.main_reactor->wait(SwooleG.main_reactor, NULL);
//clear pipe buffer
swWorker_clean();
//worker shutdown
swWorker_onStop(serv);
return SW_OK;
}
void swWorker_onStart(swServer *serv)
{
/**
* Release other worker process
*/
swWorker *worker;
if (SwooleWG.id >= serv->worker_num)
{
SwooleG.process_type = SW_PROCESS_TASKWORKER;
}
else
{
SwooleG.process_type = SW_PROCESS_WORKER;
}
SwooleWG.worker = swServer_get_worker(serv, SwooleWG.id);
int i;
for (i = 0; i < serv->worker_num + SwooleG.task_worker_num; i++)
{
worker = swServer_get_worker(serv, i);
if (SwooleWG.id == i)
{
continue;
}
else
{
swWorker_free(worker);
}
if (swIsWorker())
{
swSetNonBlock(worker->pipe_master);
}
}
if (serv->onWorkerStart)
{
serv->onWorkerStart(serv, SwooleWG.id);
}
}
int swFactoryProcess_start(swFactory *factory)
{
if (swFactory_check_callback(factory) < 0)
{
swWarn("swFactory_check_callback failed");
return SW_ERR;
}
int i;
swServer *serv = factory->ptr;
swWorker *worker;
for (i = 0; i < serv->worker_num; i++)
{
worker = swServer_get_worker(serv, i);
if (swWorker_create(worker) < 0)
{
return SW_ERR;
}
}
serv->reactor_pipe_num = serv->worker_num / serv->reactor_num;
//必须先启动manager进程组，否则会带线程fork
if (swFactoryProcess_manager_start(factory) < 0)
{
swWarn("swFactoryProcess_manager_start failed.");
return SW_ERR;
}
if (serv->ipc_mode == SW_IPC_MSGQUEUE)
{
swQueueMsg_set_blocking(&serv->read_queue, 1);
//tcp & message queue require writer pthread
if (serv->have_tcp_sock == 1)
{
int ret = swFactoryProcess_writer_start(factory);
if (ret < 0)
{
return SW_ERR;
}
}
}
//主进程需要设置为直写模式
factory->finish = swFactory_finish;
return SW_OK;
}
void swWorker_clean(void)
{
int i;
swServer *serv = SwooleG.serv;
swWorker *worker;
for (i = 0; i < serv->worker_num + SwooleG.task_worker_num; i++)
{
worker = swServer_get_worker(serv, i);
if (worker->pipe_worker)
{
swReactor_wait_write_buffer(SwooleG.main_reactor, worker->pipe_worker);
}
if (worker->pipe_master)
{
swReactor_wait_write_buffer(SwooleG.main_reactor, worker->pipe_master);
}
}
}
/**
* worker: send to client
*/
int swFactoryProcess_finish(swFactory *factory, swSendData *resp)
{
int ret, sendn, count;
swServer *serv = factory->ptr;
int fd = resp->info.fd;
//unix dgram
if (resp->info.type == SW_EVENT_UNIX_DGRAM)
{
socklen_t len;
struct sockaddr_un addr_un;
int from_sock = resp->info.from_fd;
addr_un.sun_family = AF_UNIX;
memcpy(addr_un.sun_path, resp->sun_path, resp->sun_path_len);
len = sizeof(addr_un);
ret = swSendto(from_sock, resp->data, resp->info.len, 0, (struct sockaddr *) &addr_un, len);
goto finish;
}
//UDP pacakge
else if (resp->info.type == SW_EVENT_UDP || resp->info.type == SW_EVENT_UDP6)
{
ret = swServer_udp_send(serv, resp);
goto finish;
}
//swQueue_data for msg queue
struct
{
long pti;
swEventData _send;
} sdata;
//for message queue
sdata.pti = (SwooleWG.id % serv->writer_num) + 1;
swConnection *conn = swServer_connection_get(serv, fd);
if (conn == NULL || conn->active == 0)
{
swWarn("connection[%d] not found.", fd);
return SW_ERR;
}
sdata._send.info.fd = fd;
sdata._send.info.type = resp->info.type;
swWorker *worker = swServer_get_worker(serv, SwooleWG.id);
/**
* Big response, use shared memory
*/
if (resp->length > 0)
{
int64_t wait_reactor;
/**
* Storage is in use right now, wait notify.
*/
if (worker->store.lock == 1)
{
worker->notify->read(worker->notify, &wait_reactor, sizeof(wait_reactor));
}
swPackage_response response;
response.length = resp->length;
response.worker_id = SwooleWG.id;
//swWarn("BigPackage, length=%d|worker_id=%d", response.length, response.worker_id);
sdata._send.info.from_fd = SW_RESPONSE_BIG;
sdata._send.info.len = sizeof(response);
memcpy(sdata._send.data, &response, sizeof(response));
/**
* Lock the worker storage
*/
worker->store.lock = 1;
memcpy(worker->store.ptr, resp->data, resp->length);
}
else
{
//copy data
memcpy(sdata._send.data, resp->data, resp->info.len);
sdata._send.info.len = resp->info.len;
sdata._send.info.from_fd = SW_RESPONSE_SMALL;
}
#if SW_REACTOR_SCHEDULE == 2
sdata._send.info.from_id = fd % serv->reactor_num;
#else
sdata._send.info.from_id = conn->from_id;
#endif
sendn = sdata._send.info.len + sizeof(resp->info);
//swWarn("send: sendn=%d|type=%d|content=%s", sendn, resp->info.type, resp->data);
swTrace("[Worker]input_queue[%ld]->in| fd=%d", sdata.pti, fd);
for (count = 0; count < SW_WORKER_SENDTO_COUNT; count++)
{
if (serv->ipc_mode == SW_IPC_MSGQUEUE)
{
ret = serv->write_queue.in(&serv->write_queue, (swQueue_data *) &sdata, sendn);
}
else
{
int master_pipe = swWorker_get_write_pipe(serv, fd);
//swWarn("send to reactor. fd=%d|pipe=%d|reactor_id=%d|reactor_pipe_num=%d", fd, master_pipe, conn->from_id, serv->reactor_pipe_num);
ret = write(master_pipe, &sdata._send, sendn);
#ifdef SW_WORKER_WAIT_PIPE
if (ret < 0 && errno == EAGAIN)
{
/**
* Wait pipe can be written.
*/
if (swSocket_wait(master_pipe, SW_WORKER_WAIT_TIMEOUT, SW_EVENT_WRITE) == SW_OK)
{
continue;
}
else
{
goto finish;
}
}
#endif
}
//swTraceLog("wt_queue->in: fd=%d|from_id=%d|data=%s|ret=%d|errno=%d", sdata._send.info.fd, sdata._send.info.from_id, sdata._send.data, ret, errno);
if (ret >= 0)
{
break;
}
else if (errno == EINTR)
{
continue;
}
else if (errno == EAGAIN)
{
swYield();
}
else
{
break;
}
}
finish:
if (ret < 0)
{
swWarn("sendto to reactor failed. Error: %s [%d]", strerror(errno), errno);
}
return ret;
}
//create worker child proccess
static int swFactoryProcess_manager_start(swFactory *factory)
{
swFactoryProcess *object = factory->object;
int i, pid, ret;
int reactor_pti;
swServer *serv = factory->ptr;
if (serv->ipc_mode == SW_IPC_MSGQUEUE)
{
//读数据队列
if (swQueueMsg_create(&serv->read_queue, 1, serv->message_queue_key, 1) < 0)
{
swError("[Master] swPipeMsg_create[In] fail. Error: %s [%d]", strerror(errno), errno);
return SW_ERR;
}
//为TCP创建写队列
if (serv->have_tcp_sock == 1)
{
//写数据队列
if (swQueueMsg_create(&serv->write_queue, 1, serv->message_queue_key + 1, 1) < 0)
{
swError("[Master] swPipeMsg_create[out] fail. Error: %s [%d]", strerror(errno), errno);
return SW_ERR;
}
}
}
else
{
object->pipes = sw_calloc(serv->worker_num, sizeof(swPipe));
if (object->pipes == NULL)
{
swError("malloc[worker_pipes] failed. Error: %s [%d]", strerror(errno), errno);
return SW_ERR;
}
//worker进程的pipes
for (i = 0; i < serv->worker_num; i++)
{
if (swPipeUnsock_create(&object->pipes[i], 1, SOCK_DGRAM) < 0)
{
return SW_ERR;
}
serv->workers[i].pipe_master = object->pipes[i].getFd(&object->pipes[i], 1);
serv->workers[i].pipe_worker = object->pipes[i].getFd(&object->pipes[i], 0);
}
}
if (SwooleG.task_worker_num > 0)
{
key_t msgqueue_key = 0;
if (SwooleG.task_ipc_mode > 0)
{
msgqueue_key = serv->message_queue_key + 2;
}
if (swProcessPool_create(&SwooleG.task_workers, SwooleG.task_worker_num, serv->task_max_request, msgqueue_key) < 0)
{
swWarn("[Master] create task_workers failed.");
return SW_ERR;
}
swWorker *worker;
for(i = 0; i < SwooleG.task_worker_num; i++)
{
worker = swServer_get_worker(serv, serv->worker_num + i);
if (swWorker_create(worker) < 0)
{
return SW_ERR;
}
}
//设置指针和回调函数
SwooleG.task_workers.ptr = serv;
SwooleG.task_workers.onTask = swTaskWorker_onTask;
SwooleG.task_workers.onWorkerStart = swTaskWorker_onWorkerStart;
SwooleG.task_workers.onWorkerStop = swTaskWorker_onWorkerStop;
}
pid = fork();
switch (pid)
{
//创建manager进程
case 0:
//创建子进程
for (i = 0; i < serv->worker_num; i++)
{
//close(worker_pipes[i].pipes[0]);
reactor_pti = (i % serv->writer_num);
serv->workers[i].reactor_id = reactor_pti;
pid = swFactoryProcess_worker_spawn(factory, i);
if (pid < 0)
{
swError("Fork worker process fail");
return SW_ERR;
}
else
{
serv->workers[i].pid = pid;
}
}
/**
* create task worker pool
*/
if (SwooleG.task_worker_num > 0)
{
swProcessPool_start(&SwooleG.task_workers);
}
//标识为管理进程
SwooleG.process_type = SW_PROCESS_MANAGER;
ret = swFactoryProcess_manager_loop(factory);
exit(ret);
break;
//主进程
default:
SwooleGS->manager_pid = pid;
break;
case -1:
swError("fork() failed.");
return SW_ERR;
}
return SW_OK;
}
/**
* worker main loop
*/
int swWorker_loop(swFactory *factory, int worker_id)
{
swServer *serv = factory->ptr;
#ifndef SW_WORKER_USE_SIGNALFD
SwooleG.use_signalfd = 0;
#endif
//worker_id
SwooleWG.id = worker_id;
SwooleG.pid = getpid();
//signal init
swWorker_signal_init();
swWorker *worker = swServer_get_worker(serv, worker_id);
swServer_worker_init(serv, worker);
SwooleG.main_reactor = sw_malloc(sizeof(swReactor));
if (SwooleG.main_reactor == NULL)
{
swError("[Worker] malloc for reactor failed.");
return SW_ERR;
}
if (swReactor_create(SwooleG.main_reactor, SW_REACTOR_MAXEVENTS) < 0)
{
swError("[Worker] create worker_reactor failed.");
return SW_ERR;
}
int pipe_worker = serv->workers[worker_id].pipe_worker;
swSetNonBlock(pipe_worker);
SwooleG.main_reactor->ptr = serv;
SwooleG.main_reactor->add(SwooleG.main_reactor, pipe_worker, SW_FD_PIPE | SW_EVENT_READ);
SwooleG.main_reactor->setHandle(SwooleG.main_reactor, SW_FD_PIPE, swWorker_onPipeReceive);
SwooleG.main_reactor->setHandle(SwooleG.main_reactor, SW_FD_PIPE | SW_FD_WRITE, swReactor_onWrite);
if (serv->max_request < 1)
{
SwooleWG.run_always = 1;
}
else
{
worker_task_num = serv->max_request;
if (worker_task_num > 10)
{
worker_task_num += swRandom(worker_id);
}
}
swWorker_onStart(serv);
#ifdef HAVE_SIGNALFD
if (SwooleG.use_signalfd)
{
swSignalfd_setup(SwooleG.main_reactor);
}
#endif
SwooleG.main_reactor->wait(SwooleG.main_reactor, NULL);
swWorker_onStop(serv);
return SW_OK;
}
/**
* worker: send to client
*/
static int swFactoryProcess_finish(swFactory *factory, swSendData *resp)
{
int ret, sendn;
swServer *serv = factory->ptr;
int fd = resp->info.fd;
swConnection *conn = swServer_connection_verify(serv, fd);
if (!conn)
{
swWarn("session#%d does not exist.", fd);
return SW_ERR;
}
else if ((conn->closed || conn->removed) && resp->info.type != SW_EVENT_CLOSE)
{
int _len = resp->length > 0 ? resp->length : resp->info.len;
swWarn("send %d byte failed, because session#%d is closed.", _len, fd);
return SW_ERR;
}
else if (conn->overflow)
{
swWarn("send failed, session#%d output buffer has been overflowed.", fd);
return SW_ERR;
}
swEventData ev_data;
ev_data.info.fd = fd;
ev_data.info.type = resp->info.type;
swWorker *worker = swServer_get_worker(serv, SwooleWG.id);
/**
* Big response, use shared memory
*/
if (resp->length > 0)
{
if (worker->send_shm == NULL)
{
swWarn("send failed, data is too big.");
return SW_ERR;
}
swPackage_response response;
worker->lock.lock(&worker->lock);
response.length = resp->length;
response.worker_id = SwooleWG.id;
//swWarn("BigPackage, length=%d|worker_id=%d", response.length, response.worker_id);
ev_data.info.from_fd = SW_RESPONSE_BIG;
ev_data.info.len = sizeof(response);
memcpy(ev_data.data, &response, sizeof(response));
memcpy(worker->send_shm, resp->data, resp->length);
}
else
{
//copy data
memcpy(ev_data.data, resp->data, resp->info.len);
ev_data.info.len = resp->info.len;
ev_data.info.from_fd = SW_RESPONSE_SMALL;
}
ev_data.info.from_id = conn->from_id;
sendn = ev_data.info.len + sizeof(resp->info);
swTrace("[Worker] send: sendn=%d|type=%d|content=%s", sendn, resp->info.type, resp->data);
ret = swWorker_send2reactor(&ev_data, sendn, fd);
if (ret < 0)
{
swWarn("sendto to reactor failed. Error: %s [%d]", strerror(errno), errno);
}
return ret;
}
void swWorker_onStart(swServer *serv)
{
/**
* Release other worker process
*/
swWorker *worker;
if (SwooleWG.id >= serv->worker_num)
{
SwooleG.process_type = SW_PROCESS_TASKWORKER;
}
else
{
SwooleG.process_type = SW_PROCESS_WORKER;
}
int is_root = !geteuid();
struct passwd *passwd = NULL;
struct group *group = NULL;
if (is_root)
{
//get group info
if (SwooleG.group)
{
group = getgrnam(SwooleG.group);
if (!group)
{
swWarn("get group [%s] info failed.", SwooleG.group);
}
}
//get user info
if (SwooleG.user)
{
passwd = getpwnam(SwooleG.user);
if (!passwd)
{
swWarn("get user [%s] info failed.", SwooleG.user);
}
}
//chroot
if (SwooleG.chroot)
{
if (0 > chroot(SwooleG.chroot))
{
swSysError("chroot to [%s] failed.", SwooleG.chroot);
}
}
//set process group
if (SwooleG.group && group)
{
if (setgid(group->gr_gid) < 0)
{
swSysError("setgid to [%s] failed.", SwooleG.group);
}
}
//set process user
if (SwooleG.user && passwd)
{
if (setuid(passwd->pw_uid) < 0)
{
swSysError("setuid to [%s] failed.", SwooleG.user);
}
}
}
SwooleWG.worker = swServer_get_worker(serv, SwooleWG.id);
int i;
for (i = 0; i < serv->worker_num + SwooleG.task_worker_num; i++)
{
worker = swServer_get_worker(serv, i);
if (SwooleWG.id == i)
{
continue;
}
else
{
swWorker_free(worker);
}
if (swIsWorker())
{
swSetNonBlock(worker->pipe_master);
}
}
if (serv->onWorkerStart)
{
serv->onWorkerStart(serv, SwooleWG.id);
}
}
/**
* Send the task result to worker
*/
int swTaskWorker_finish(swServer *serv, char *data, int data_len, int flags)
{
swEventData buf;
if (SwooleG.task_worker_num < 1)
{
swWarn("cannot use task/finish, because no set serv->task_worker_num.");
return SW_ERR;
}
int ret;
//for swoole_server_task
if (swTask_type(current_task) & SW_TASK_NONBLOCK)
{
buf.info.type = SW_EVENT_FINISH;
buf.info.fd = current_task->info.fd;
swTask_type(&buf) = flags;
//write to file
if (data_len >= SW_IPC_MAX_SIZE - sizeof(buf.info))
{
if (swTaskWorker_large_pack(&buf, data, data_len) < 0 )
{
swWarn("large task pack failed()");
return SW_ERR;
}
}
else
{
memcpy(buf.data, data, data_len);
buf.info.len = data_len;
}
uint16_t target_worker_id = current_task->info.from_id;
swWorker *worker = swServer_get_worker(serv, target_worker_id);
ret = swWorker_send2worker(worker, &buf, sizeof(buf.info) + buf.info.len, SW_PIPE_MASTER);
}
else
{
uint64_t flag = 1;
uint16_t worker_id = current_task->info.from_id;
/**
* Use worker shm store the result
*/
swEventData *result = &(SwooleG.task_result[worker_id]);
swPipe *task_notify_pipe = &(SwooleG.task_notify[worker_id]);
result->info.type = SW_EVENT_FINISH;
result->info.fd = current_task->info.fd;
swTask_type(result) = flags;
if (data_len >= SW_IPC_MAX_SIZE - sizeof(buf.info))
{
if (swTaskWorker_large_pack(result, data, data_len) < 0)
{
swWarn("large task pack failed()");
return SW_ERR;
}
}
else
{
memcpy(result->data, data, data_len);
result->info.len = data_len;
}
while (1)
{
ret = task_notify_pipe->write(task_notify_pipe, &flag, sizeof(flag));
#ifdef HAVE_KQUEUE
if (errno == EAGAIN || errno == ENOBUFS)
#else
if (errno == EAGAIN)
#endif
{
if (swSocket_wait(task_notify_pipe->getFd(task_notify_pipe, 1), -1, SW_EVENT_WRITE) == 0)
{
continue;
}
}
break;
}
}
if (ret < 0)
{
swWarn("TaskWorker: send result to worker failed. Error: %s[%d]", strerror(errno), errno);
}
return ret;
}
int swConnection_send_string_buffer(swConnection *conn)
{
int ret;
swString *buffer = conn->object;
swFactory *factory = SwooleG.factory;
swDispatchData task;
task.data.info.fd = conn->fd;
task.data.info.from_id = conn->from_id;
#ifdef SW_USE_RINGBUFFER
swServer *serv = SwooleG.serv;
int target_worker_id = swServer_worker_schedule(serv, conn->fd);
swWorker *worker = swServer_get_worker(serv, target_worker_id);
swMemoryPool *pool = worker->pool_input;
swPackage package;
package.length = buffer->length;
while (1)
{
package.data = pool->alloc(pool, buffer->length);
if (package.data == NULL)
{
swYield();
swWarn("reactor memory pool full.");
continue;
}
break;
}
task.data.info.type = SW_EVENT_PACKAGE;
task.data.info.len = sizeof(package);
task.target_worker_id = target_worker_id;
//swoole_dump_bin(package.data, 's', buffer->length);
memcpy(package.data, buffer->str, buffer->length);
memcpy(task.data.data, &package, sizeof(package));
ret = factory->dispatch(factory, &task);
#else
int send_n = buffer->length;
task.data.info.type = SW_EVENT_PACKAGE_START;
task.target_worker_id = -1;
/**
* lock target
*/
SwooleTG.factory_lock_target = 1;
void *send_ptr = buffer->str;
do
{
if (send_n > SW_BUFFER_SIZE)
{
task.data.info.len = SW_BUFFER_SIZE;
memcpy(task.data.data, send_ptr, SW_BUFFER_SIZE);
}
else
{
task.data.info.type = SW_EVENT_PACKAGE_END;
task.data.info.len = send_n;
memcpy(task.data.data, send_ptr, send_n);
}
swTrace("dispatch, type=%d|len=%d\n", _send.info.type, _send.info.len);
ret = factory->dispatch(factory, &task);
//TODO: 处理数据失败，数据将丢失
if (ret < 0)
{
swWarn("factory->dispatch failed.");
}
send_n -= task.data.info.len;
send_ptr += task.data.info.len;
}
while (send_n > 0);
/**
* unlock
*/
SwooleTG.factory_target_worker = -1;
SwooleTG.factory_lock_target = 0;
#endif
return ret;
}
/**
* worker main loop
*/
int swWorker_loop(swFactory *factory, int worker_id)
{
swServer *serv = factory->ptr;
#ifndef SW_WORKER_USE_SIGNALFD
SwooleG.use_signalfd = 0;
#elif defined(HAVE_SIGNALFD)
SwooleG.use_signalfd = 1;
#endif
//timerfd
#ifdef HAVE_TIMERFD
SwooleG.use_timerfd = 1;
#endif
//worker_id
SwooleWG.id = worker_id;
SwooleWG.request_count = 0;
SwooleG.pid = getpid();
//signal init
swWorker_signal_init();
swWorker *worker = swServer_get_worker(serv, worker_id);
swServer_worker_init(serv, worker);
SwooleG.main_reactor = sw_malloc(sizeof(swReactor));
if (SwooleG.main_reactor == NULL)
{
swError("[Worker] malloc for reactor failed.");
return SW_ERR;
}
if (swReactor_create(SwooleG.main_reactor, SW_REACTOR_MAXEVENTS) < 0)
{
swError("[Worker] create worker_reactor failed.");
return SW_ERR;
}
serv->workers[worker_id].status = SW_WORKER_IDLE;
int pipe_worker = serv->workers[worker_id].pipe_worker;
swSetNonBlock(pipe_worker);
SwooleG.main_reactor->ptr = serv;
SwooleG.main_reactor->add(SwooleG.main_reactor, pipe_worker, SW_FD_PIPE | SW_EVENT_READ);
SwooleG.main_reactor->setHandle(SwooleG.main_reactor, SW_FD_PIPE, swWorker_onPipeReceive);
SwooleG.main_reactor->setHandle(SwooleG.main_reactor, SW_FD_PIPE | SW_FD_WRITE, swReactor_onWrite);
/**
* set pipe buffer size
*/
int i;
swConnection *pipe_socket;
for (i = 0; i < serv->worker_num + SwooleG.task_worker_num; i++)
{
worker = swServer_get_worker(serv, i);
pipe_socket = swReactor_get(SwooleG.main_reactor, worker->pipe_master);
pipe_socket->buffer_size = serv->pipe_buffer_size;
pipe_socket = swReactor_get(SwooleG.main_reactor, worker->pipe_worker);
pipe_socket->buffer_size = serv->pipe_buffer_size;
}
swWorker_onStart(serv);
#ifdef HAVE_SIGNALFD
if (SwooleG.use_signalfd)
{
swSignalfd_setup(SwooleG.main_reactor);
}
#endif
//main loop
SwooleG.main_reactor->wait(SwooleG.main_reactor, NULL);
//clear pipe buffer
swWorker_clean();
//worker shutdown
swWorker_onStop(serv);
return SW_OK;
}
/**
* Send the task result to worker
*/
int swTaskWorker_finish(swServer *serv, char *data, int data_len, int flags)
{
swEventData buf;
if (!current_task)
{
swWarn("cannot use finish in worker");
return SW_ERR;
}
if (serv->task_worker_num < 1)
{
swWarn("cannot use task/finish, because no set serv->task_worker_num.");
return SW_ERR;
}
if (current_task->info.type == SW_EVENT_PIPE_MESSAGE)
{
swWarn("task/finish is not supported in onPipeMessage callback.");
return SW_ERR;
}
uint16_t source_worker_id = current_task->info.from_id;
swWorker *worker = swServer_get_worker(serv, source_worker_id);
if (worker == NULL)
{
swWarn("invalid worker_id[%d].", source_worker_id);
return SW_ERR;
}
int ret;
//for swoole_server_task
if (swTask_type(current_task) & SW_TASK_NONBLOCK)
{
buf.info.type = SW_EVENT_FINISH;
buf.info.fd = current_task->info.fd;
//callback function
if (swTask_type(current_task) & SW_TASK_CALLBACK)
{
flags |= SW_TASK_CALLBACK;
}
else if (swTask_type(current_task) & SW_TASK_COROUTINE)
{
flags |= SW_TASK_COROUTINE;
}
swTask_type(&buf) = flags;
//write to file
if (data_len >= SW_IPC_MAX_SIZE - sizeof(buf.info))
{
if (swTaskWorker_large_pack(&buf, data, data_len) < 0 )
{
swWarn("large task pack failed()");
return SW_ERR;
}
}
else
{
memcpy(buf.data, data, data_len);
buf.info.len = data_len;
}
if (worker->pool->use_socket && worker->pool->stream->last_connection > 0)
{
int32_t _len = htonl(data_len);
ret = swSocket_write_blocking(worker->pool->stream->last_connection, (void *) &_len, sizeof(_len));
if (ret > 0)
{
ret = swSocket_write_blocking(worker->pool->stream->last_connection, data, data_len);
}
}
else
{
ret = swWorker_send2worker(worker, &buf, sizeof(buf.info) + buf.info.len, SW_PIPE_MASTER);
}
}
else
{
uint64_t flag = 1;
/**
* Use worker shm store the result
*/
swEventData *result = &(serv->task_result[source_worker_id]);
swPipe *task_notify_pipe = &(serv->task_notify[source_worker_id]);
//lock worker
worker->lock.lock(&worker->lock);
if (swTask_type(current_task) & SW_TASK_WAITALL)
{
sw_atomic_t *finish_count = (sw_atomic_t*) result->data;
char *_tmpfile = result->data + 4;
int fd = open(_tmpfile, O_APPEND | O_WRONLY);
if (fd >= 0)
{
buf.info.type = SW_EVENT_FINISH;
buf.info.fd = current_task->info.fd;
swTask_type(&buf) = flags;
//result pack
if (data_len >= SW_IPC_MAX_SIZE - sizeof(buf.info))
{
if (swTaskWorker_large_pack(&buf, data, data_len) < 0)
{
swWarn("large task pack failed()");
buf.info.len = 0;
}
}
else
{
buf.info.len = data_len;
memcpy(buf.data, data, data_len);
}
//write to tmpfile
if (swoole_sync_writefile(fd, &buf, sizeof(buf.info) + buf.info.len) < 0)
{
swSysError("write(%s, %ld) failed.", _tmpfile, sizeof(buf.info) + buf.info.len);
}
sw_atomic_fetch_add(finish_count, 1);
close(fd);
}
}
else
{
result->info.type = SW_EVENT_FINISH;
result->info.fd = current_task->info.fd;
swTask_type(result) = flags;
if (data_len >= SW_IPC_MAX_SIZE - sizeof(buf.info))
{
if (swTaskWorker_large_pack(result, data, data_len) < 0)
{
//unlock worker
worker->lock.unlock(&worker->lock);
swWarn("large task pack failed()");
return SW_ERR;
}
}
else
{
memcpy(result->data, data, data_len);
result->info.len = data_len;
}
}
//unlock worker
worker->lock.unlock(&worker->lock);
while (1)
{
ret = task_notify_pipe->write(task_notify_pipe, &flag, sizeof(flag));
#ifdef HAVE_KQUEUE
if (ret < 0 && (errno == EAGAIN || errno == ENOBUFS))
#else
if (ret < 0 && errno == EAGAIN)
#endif
{
if (swSocket_wait(task_notify_pipe->getFd(task_notify_pipe, 1), -1, SW_EVENT_WRITE) == 0)
{
continue;
}
}
break;
}
}
if (ret < 0)
{
swWarn("TaskWorker: send result to worker failed. Error: %s[%d]", strerror(errno), errno);
}
return ret;
}
/**
* worker: send to client
*/
int swFactoryProcess_finish(swFactory *factory, swSendData *resp)
{
int ret, sendn;
swServer *serv = factory->ptr;
int fd = resp->info.fd;
//unix dgram
if (resp->info.type == SW_EVENT_UNIX_DGRAM)
{
socklen_t len;
struct sockaddr_un addr_un;
int from_sock = resp->info.from_fd;
addr_un.sun_family = AF_UNIX;
memcpy(addr_un.sun_path, resp->sun_path, resp->sun_path_len);
len = sizeof(addr_un);
ret = swSendto(from_sock, resp->data, resp->info.len, 0, (struct sockaddr *) &addr_un, len);
goto finish;
}
//UDP pacakge
else if (resp->info.type == SW_EVENT_UDP || resp->info.type == SW_EVENT_UDP6)
{
return swServer_udp_send(serv, resp);
}
//for message queue
swEventData_overflow sdata;
sdata.pti = (SwooleWG.id % serv->writer_num) + 1;
swConnection *conn = swServer_connection_get(serv, fd);
if (conn == NULL || conn->active == 0)
{
swWarn("connection[%d] not found.", fd);
return SW_ERR;
}
sdata._send.info.fd = fd;
sdata._send.info.type = resp->info.type;
swWorker *worker = swServer_get_worker(serv, SwooleWG.id);
/**
* Big response, use shared memory
*/
if (resp->length > 0)
{
swPackage_response response;
worker->lock.lock(&worker->lock);
response.length = resp->length;
response.worker_id = SwooleWG.id;
//swWarn("BigPackage, length=%d|worker_id=%d", response.length, response.worker_id);
sdata._send.info.from_fd = SW_RESPONSE_BIG;
sdata._send.info.len = sizeof(response);
memcpy(sdata._send.data, &response, sizeof(response));
memcpy(worker->send_shm, resp->data, resp->length);
}
else
{
//copy data
memcpy(sdata._send.data, resp->data, resp->info.len);
sdata._send.info.len = resp->info.len;
sdata._send.info.from_fd = SW_RESPONSE_SMALL;
}
#if SW_REACTOR_SCHEDULE == 2
sdata._send.info.from_id = fd % serv->reactor_num;
#else
sdata._send.info.from_id = conn->from_id;
#endif
sendn = sdata._send.info.len + sizeof(resp->info);
//swWarn("send: sendn=%d|type=%d|content=%s", sendn, resp->info.type, resp->data);
swTrace("[Worker]input_queue[%ld]->in| fd=%d", sdata.pti, fd);
ret = swWorker_send2reactor(&sdata, sendn, fd);
finish:
if (ret < 0)
{
swWarn("sendto to reactor failed. Error: %s [%d]", strerror(errno), errno);
}
return ret;
}
/**
* worker: send to client
*/
static int swFactoryProcess_finish(swFactory *factory, swSendData *resp)
{
int ret, sendn;
swServer *serv = factory->ptr;
int session_id = resp->info.fd;
swConnection *conn;
if (resp->info.type != SW_EVENT_CLOSE)
{
conn = swServer_connection_verify(serv, session_id);
}
else
{
conn = swServer_connection_verify_no_ssl(serv, session_id);
}
if (!conn)
{
swoole_error_log(SW_LOG_NOTICE, SW_ERROR_SESSION_NOT_EXIST, "connection[fd=%d] does not exists.", session_id);
return SW_ERR;
}
else if ((conn->closed || conn->removed) && resp->info.type != SW_EVENT_CLOSE)
{
int _len = resp->length > 0 ? resp->length : resp->info.len;
swoole_error_log(SW_LOG_NOTICE, SW_ERROR_SESSION_CLOSED, "send %d byte failed, because connection[fd=%d] is closed.", _len, session_id);
return SW_ERR;
}
else if (conn->overflow)
{
swoole_error_log(SW_LOG_WARNING, SW_ERROR_OUTPUT_BUFFER_OVERFLOW, "send failed, connection[fd=%d] output buffer has been overflowed.", session_id);
return SW_ERR;
}
swEventData ev_data;
ev_data.info.fd = session_id;
ev_data.info.type = resp->info.type;
swWorker *worker = swServer_get_worker(serv, SwooleWG.id);
/**
* Big response, use shared memory
*/
if (resp->length > 0)
{
if (worker == NULL || worker->send_shm == NULL)
{
goto pack_data;
}
//worker process
if (SwooleG.main_reactor)
{
int _pipe_fd = swWorker_get_send_pipe(serv, session_id, conn->from_id);
swConnection *_pipe_socket = swReactor_get(SwooleG.main_reactor, _pipe_fd);
//cannot use send_shm
if (!swBuffer_empty(_pipe_socket->out_buffer))
{
pack_data:
if (swTaskWorker_large_pack(&ev_data, resp->data, resp->length) < 0)
{
return SW_ERR;
}
ev_data.info.from_fd = SW_RESPONSE_TMPFILE;
goto send_to_reactor_thread;
}
}
swPackage_response response;
response.length = resp->length;
response.worker_id = SwooleWG.id;
ev_data.info.from_fd = SW_RESPONSE_SHM;
ev_data.info.len = sizeof(response);
memcpy(ev_data.data, &response, sizeof(response));
swTrace("[Worker] big response, length=%d|worker_id=%d", response.length, response.worker_id);
worker->lock.lock(&worker->lock);
memcpy(worker->send_shm, resp->data, resp->length);
}
else
{
//copy data
memcpy(ev_data.data, resp->data, resp->info.len);
ev_data.info.len = resp->info.len;
ev_data.info.from_fd = SW_RESPONSE_SMALL;
}
send_to_reactor_thread: ev_data.info.from_id = conn->from_id;
sendn = ev_data.info.len + sizeof(resp->info);
swTrace("[Worker] send: sendn=%d|type=%d|content=%s", sendn, resp->info.type, resp->data);
ret = swWorker_send2reactor(&ev_data, sendn, session_id);
if (ret < 0)
{
swWarn("sendto to reactor failed. Error: %s [%d]", strerror(errno), errno);
}
return ret;
}
int swConnection_send_in_buffer(swConnection *conn)
{
swDispatchData task;
swFactory *factory = SwooleG.factory;
task.data.info.fd = conn->fd;
task.data.info.from_id = conn->from_id;
swBuffer *buffer = conn->in_buffer;
swBuffer_trunk *trunk = swBuffer_get_trunk(buffer);
#ifdef SW_USE_RINGBUFFER
swServer *serv = SwooleG.serv;
uint16_t target_worker_id = swServer_worker_schedule(serv, conn->fd);
swWorker *worker = swServer_get_worker(serv, target_worker_id);
swMemoryPool *pool = worker->pool_input;
swPackage package;
package.length = 0;
while (1)
{
package.data = pool->alloc(pool, buffer->length);
if (package.data == NULL)
{
swYield();
swWarn("reactor memory pool full.");
continue;
}
break;
}
task.data.info.type = SW_EVENT_PACKAGE;
while (trunk != NULL)
{
task.data.info.len = trunk->length;
memcpy(package.data + package.length, trunk->store.ptr, trunk->length);
package.length += trunk->length;
swBuffer_pop_trunk(buffer, trunk);
trunk = swBuffer_get_trunk(buffer);
}
task.data.info.len = sizeof(package);
task.target_worker_id = target_worker_id;
memcpy(task.data.data, &package, sizeof(package));
//swWarn("[ReactorThread] copy_n=%d", package.length);
return factory->dispatch(factory, &task);
#else
int ret;
task.data.info.type = SW_EVENT_PACKAGE_START;
task.target_worker_id = -1;
/**
* lock target
*/
SwooleTG.factory_lock_target = 1;
while (trunk != NULL)
{
task.data.info.len = trunk->length;
memcpy(task.data.data, trunk->store.ptr, task.data.info.len);
//package end
if (trunk->next == NULL)
{
task.data.info.type = SW_EVENT_PACKAGE_END;
}
ret = factory->dispatch(factory, &task);
//TODO: 处理数据失败，数据将丢失
if (ret < 0)
{
swWarn("factory->dispatch() failed.");
}
swBuffer_pop_trunk(buffer, trunk);
trunk = swBuffer_get_trunk(buffer);
swTrace("send2worker[trunk_num=%d][type=%d]\n", buffer->trunk_num, _send.info.type);
}
/**
* unlock
*/
SwooleTG.factory_target_worker = -1;
SwooleTG.factory_lock_target = 0;
#endif
return SW_OK;
}
void swWorker_onStart(swServer *serv)
{
/**
* Release other worker process
*/
int i;
swWorker *worker;
if (SwooleWG.id >= serv->worker_num)
{
SwooleG.process_type = SW_PROCESS_TASKWORKER;
}
else
{
SwooleG.process_type = SW_PROCESS_WORKER;
}
SwooleWG.worker = swServer_get_worker(serv, SwooleWG.id);
if (swIsWorker())
{
int maxfd;
if (SwooleG.task_worker_num > 0)
{
maxfd = SwooleG.task_workers.workers[SwooleG.task_worker_num - 1].pipe_master + 1;
}
else
{
maxfd = serv->workers[serv->worker_num - 1].pipe_master + 1;
}
SwooleWG.fd_map = swArray_new(maxfd, sizeof(swPipe *), 0);
}
for (i = 0; i < serv->worker_num + SwooleG.task_worker_num; i++)
{
worker = swServer_get_worker(serv, i);
if (SwooleWG.id == i)
{
continue;
}
else
{
swWorker_free(worker);
}
if (SwooleWG.id < serv->worker_num && i < serv->worker_num)
{
close(worker->pipe_worker);
}
if (swIsWorker())
{
swSetNonBlock(worker->pipe_master);
swArray_store(SwooleWG.fd_map, worker->pipe_master, &worker->pipe_object);
worker->pipe_object->pipe_used = worker->pipe_master;
}
}
if (serv->onWorkerStart)
{
serv->onWorkerStart(serv, SwooleWG.id);
}
}
