bool chat_client_connect() {
getplayername();
if (!ircThreadInstance) {
ircThreadInstance = new irc_thread__parm();
}
int temporarySocket = tcpsocket();
struct sockaddr_in addr;
memset(&addr, 0, sizeof(addr));
int ip = resolv(Form1->getChatHost().c_str());
addr.sin_addr.s_addr = ip;
addr.sin_port = htons(Form1->getChatPort());
addr.sin_family = AF_INET;
int connectRes = connect(temporarySocket, (struct sockaddr *)&addr, sizeof(struct sockaddr_in));
if( connectRes >= 0 && temporarySocket) {
ircThreadInstance->sd = temporarySocket;
CreateThread(0, 0, irc_ThreadProc, ircThreadInstance, 0, 0);
if (!rReconnectWatchThread_started) {
rReconnectWatchThread_started = 1;
CreateThread(0, 0, irc_ThreadProc_ReconnectWatchThread, 0, 0, 0);
}
return true;
} else {
closesocket(temporarySocket);
}
return false;
}
int client_socket (char * host, char * port) {
int status;
int lsock = tcpsocket();
if (lsock<0) {
logging(LOG_ERROR, "can't create socket !!!");
return -1;
}
if (tcpnonblock(lsock)<0) {
logging(LOG_ERROR, "set nonblock, error: %m");
tcpclose(lsock);
return -1;
}
status = tcpstrconnect(lsock, host, port);
if (status<0) {
logging(LOG_WARN, "connect failed, error: %m");
tcpclose(lsock);
return -1;
}
if (status==0) { // connected immediately
logging(LOG_INFO,"connected immediately");
tcpnodelay(lsock);
} else {
logging(LOG_INFO,"connecting to %s:%s...", host, port);
}
return lsock;
}
void* ppfs_fsinit( struct fuse_conn_info* conn ) { //connect to MDS
fprintf(stderr, "ppfs_fsinit\n");
if((fd = tcpsocket())<0) {
fprintf(stderr, "can't create socket\n");
exit(1);
} else {
fprintf(stderr, "fd:%d\n", fd);
}
tcpnodelay(fd);
if(tcpstrconnect(fd, ip, MDS_PORT_STR)<0) {
fprintf(stderr, "can't connect to MDS (%s:%s)\n", ip, port);
fd = -1;
exit(1);
}
fprintf(stderr, "ppfs_init create socket: %u\n", fd);
local_mds.sockfd = fd;
tcpgetpeer(fd,&(local_mds.peerip),NULL);
remote_mds.sockfd = -1;
remote_mds.peerip = -1;
}
int matomlserv_init(void) {
ListenHost = cfg_getstr("MATOML_LISTEN_HOST","*");
ListenPort = cfg_getstr("MATOML_LISTEN_PORT","9419");
lsock = tcpsocket();
if (lsock<0) {
mfs_errlog(LOG_ERR,"master <-> metaloggers module: can't create socket");
return -1;
}
tcpnonblock(lsock);
tcpnodelay(lsock);
tcpreuseaddr(lsock);
if (tcpsetacceptfilter(lsock)<0 && errno!=ENOTSUP) {
mfs_errlog_silent(LOG_NOTICE,"master <-> metaloggers module: can't set accept filter");
}
if (tcpstrlisten(lsock,ListenHost,ListenPort,100)<0) {
mfs_errlog(LOG_ERR,"master <-> metaloggers module: can't listen on socket");
return -1;
}
mfs_arg_syslog(LOG_NOTICE,"master <-> metaloggers module: listen on %s:%s",ListenHost,ListenPort);
matomlservhead = NULL;
main_reloadregister(matomlserv_reload);
main_destructregister(matomlserv_term);
main_pollregister(matomlserv_desc,matomlserv_serve);
main_timeregister(TIMEMODE_SKIP_LATE,3600,0,matomlserv_status);
return 0;
}
int csserv_init(void) {
ListenHost = cfg_getstr("CSSERV_LISTEN_HOST","*");
ListenPort = cfg_getstr("CSSERV_LISTEN_PORT",DEFAULT_CS_DATA_PORT);
lsock = tcpsocket();
if (lsock<0) {
mfs_errlog(LOG_ERR,"main server module: can't create socket");
return -1;
}
tcpnonblock(lsock);
tcpnodelay(lsock);
tcpreuseaddr(lsock);
tcpresolve(ListenHost,ListenPort,&mylistenip,&mylistenport,1);
if (tcpnumlisten(lsock,mylistenip,mylistenport,100)<0) {
mfs_errlog(LOG_ERR,"main server module: can't listen on socket");
return -1;
}
if (tcpsetacceptfilter(lsock)<0 && errno!=ENOTSUP) {
mfs_errlog_silent(LOG_NOTICE,"main server module: can't set accept filter");
}
mfs_arg_syslog(LOG_NOTICE,"main server module: listen on %s:%s",ListenHost,ListenPort);
csservhead = NULL;
main_reload_register(csserv_reload);
main_destruct_register(csserv_term);
main_poll_register(csserv_desc,csserv_serve);
return 0;
}
int matomlserv_init(void) {
ListenHost = cfg_getstr("MATOML_LISTEN_HOST","*");
ListenPort = cfg_getstr("MATOML_LISTEN_PORT","9419");
lsock = tcpsocket();
if (lsock<0) {
mfs_errlog(LOG_ERR,"master <-> metaloggers module: can't create socket");
return -1;
}
tcpnonblock(lsock);
tcpnodelay(lsock);
tcpreuseaddr(lsock);
if (tcpsetacceptfilter(lsock)<0 && errno!=ENOTSUP) {
mfs_errlog_silent(LOG_NOTICE,"master <-> metaloggers module: can't set accept filter");
}
if (tcpstrlisten(lsock,ListenHost,ListenPort,100)<0) {
mfs_arg_errlog(LOG_ERR,"master <-> metaloggers module: can't listen on %s:%s",ListenHost,ListenPort);
return -1;
}
mfs_arg_syslog(LOG_NOTICE,"master <-> metaloggers module: listen on %s:%s",ListenHost,ListenPort);
matomlservhead = NULL;
ChangelogSecondsToRemember = cfg_getuint16("MATOML_LOG_PRESERVE_SECONDS",600);
if (ChangelogSecondsToRemember>3600) {
syslog(LOG_WARNING,"Number of seconds of change logs to be preserved in master is too big (%"PRIu16") - decreasing to 3600 seconds",ChangelogSecondsToRemember);
ChangelogSecondsToRemember=3600;
}
main_reloadregister(matomlserv_reload);
main_destructregister(matomlserv_term);
main_canexitregister(matomlserv_canexit);
main_pollregister(matomlserv_desc,matomlserv_serve);
main_timeregister(TIMEMODE_SKIP_LATE,3600,0,matomlserv_status);
return 0;
}
int masterproxy_init(void) {
pthread_attr_t thattr;
lsock = tcpsocket();
if (lsock<0) {
//mfs_errlog(LOG_ERR,"main master server module: can't create socket");
return -1;
}
tcpnonblock(lsock);
tcpnodelay(lsock);
// tcpreuseaddr(lsock);
if (tcpsetacceptfilter(lsock)<0 && errno!=ENOTSUP) {
// mfs_errlog_silent(LOG_NOTICE,"master proxy: can't set accept filter");
}
if (tcpstrlisten(lsock,"127.0.0.1",0,100)<0) {
// mfs_errlog(LOG_ERR,"main master server module: can't listen on socket");
tcpclose(lsock);
lsock = -1;
return -1;
}
if (tcpgetmyaddr(lsock,&proxyhost,&proxyport)<0) {
tcpclose(lsock);
lsock = -1;
return -1;
}
terminate = 0;
pthread_attr_init(&thattr);
pthread_attr_setstacksize(&thattr,0x100000);
//pthread_create(&proxythread,&thattr,masterproxy_loop,NULL);
pthread_create(&proxythread,&thattr,masterproxy_acceptor,NULL);
pthread_attr_destroy(&thattr);
return 1;
}
static int write_data_refresh_connection(uint32_t inode, uint32_t indx, uint64_t *chunkid)
{
uint32_t ip;
uint16_t port;
uint8_t status;
if (rw_sock >= 0) {
tcpclose(rw_sock);
rw_sock = -1;
}
status = fs_writechunk(&ip,&port, chunkid, inode, indx);
if (status != 0) {
fprintf(stderr, "status error\n");
return -2;
}
rw_sock = tcpsocket();
if (rw_sock < 0) {
fprintf(stderr,"can't create tcp socket\n");
return -1;
}
if (tcpnodelay(rw_sock)<0) {
fprintf(stderr,"can't set TCP_NODELAY\n");
}
if (tcpnumconnect(rw_sock, ip, port)<0) {
fprintf(stderr,"can't connect to (%d.%d.%d.%d:%u)\n",
ip/(256*256*256)%256, ip/(256*256)%256, ip/256%256, ip%256, port);
tcpclose(rw_sock);
rw_sock = -1;
return -1;
}
return 0;
}
int masterconn_initconnect(masterconn *eptr) {
int status;
if (eptr->masteraddrvalid==0) {
uint32_t mip,bip;
uint16_t mport;
if (tcpresolve(BindHost,NULL,&bip,NULL,1)<0) {
bip = 0;
}
eptr->bindip = bip;
if (tcpresolve(MasterHost,MasterPort,&mip,&mport,0)>=0) {
eptr->masterip = mip;
eptr->masterport = mport;
eptr->masteraddrvalid = 1;
} else {
mfs_arg_syslog(LOG_WARNING,"can't resolve master host/port (%s:%s)",MasterHost,MasterPort);
return -1;
}
}
eptr->sock=tcpsocket();
if (eptr->sock<0) {
mfs_errlog(LOG_WARNING,"create socket, error");
return -1;
}
if (tcpnonblock(eptr->sock)<0) {
mfs_errlog(LOG_WARNING,"set nonblock, error");
tcpclose(eptr->sock);
eptr->sock = -1;
return -1;
}
if (eptr->bindip>0) {
if (tcpnumbind(eptr->sock,eptr->bindip,0)<0) {
mfs_errlog(LOG_WARNING,"can't bind socket to given ip");
tcpclose(eptr->sock);
eptr->sock = -1;
return -1;
}
}
status = tcpnumconnect(eptr->sock,eptr->masterip,eptr->masterport);
if (status<0) {
mfs_errlog(LOG_WARNING,"connect failed, error");
tcpclose(eptr->sock);
eptr->sock = -1;
eptr->masteraddrvalid = 0;
return -1;
}
if (status==0) {
syslog(LOG_NOTICE,"connected to Master immediately");
masterconn_connected(eptr);
} else {
eptr->mode = CONNECTING;
eptr->conntime = monotonic_seconds();
syslog(LOG_NOTICE,"connecting ...");
}
return 0;
}
int masterconn_initconnect(serventry *eptr) {
int status;
if (eptr->masteraddrvalid==0) {
uint32_t mip,bip;
uint16_t mport;
if (tcpresolve(BindHost,NULL,&bip,NULL,1)>=0) {
eptr->bindip = bip;
} else {
eptr->bindip = 0;
}
if (tcpresolve(MasterHost,MasterPort,&mip,&mport,0)>=0) {
eptr->masterip = mip;
eptr->masterport = mport;
eptr->masteraddrvalid = 1;
} else {
MFSLOG(LOG_WARNING,"can't resolve master host/port (%s:%s)",MasterHost,MasterPort);
return -1;
}
}
eptr->sock=tcpsocket();
if (eptr->sock<0) {
MFSLOG(LOG_WARNING,"create socket, error: %m");
return -1;
}
if (tcpnonblock(eptr->sock)<0) {
MFSLOG(LOG_WARNING,"set nonblock, error: %m");
tcpclose(eptr->sock);
eptr->sock=-1;
return -1;
}
if (eptr->bindip>0) {
if (tcpnumbind(eptr->sock,eptr->bindip,0)<0) {
MFSLOG(LOG_WARNING,"can't bind socket to given ip: %m");
tcpclose(eptr->sock);
eptr->sock=-1;
return -1;
}
}
status = tcpnumconnect(eptr->sock,eptr->masterip,eptr->masterport);
if (status<0) {
MFSLOG(LOG_WARNING,"connect failed, error: %m");
tcpclose(eptr->sock);
eptr->sock=-1;
return -1;
}
if (status==0) {
MFSLOG(LOG_NOTICE,"connected to Master immediately");
masterconn_connected(eptr);
} else {
eptr->mode = CONNECTING;
MFSLOG(LOG_NOTICE,"connecting ...");
}
// syslog(LOG_NOTICE,"the masconn's sock is %d,the connection id is %d,masterconnsingleton's sock is %d,masterconnsingleton's id is %d",eptr->sock,eptr->connection,masterconnsingleton->sock,masterconnsingleton->connection);
return 0;
}
void matomlserv_reload(void) {
char *oldListenHost,*oldListenPort;
int newlsock;
oldListenHost = ListenHost;
oldListenPort = ListenPort;
ListenHost = cfg_getstr("MATOML_LISTEN_HOST","*");
ListenPort = cfg_getstr("MATOML_LISTEN_PORT","9419");
if (strcmp(oldListenHost,ListenHost)==0 && strcmp(oldListenPort,ListenPort)==0) {
free(oldListenHost);
free(oldListenPort);
mfs_arg_syslog(LOG_NOTICE,"master <-> metaloggers module: socket address hasn't changed (%s:%s)",ListenHost,ListenPort);
return;
}
newlsock = tcpsocket();
if (newlsock<0) {
mfs_errlog(LOG_WARNING,"master <-> metaloggers module: socket address has changed, but can't create new socket");
free(ListenHost);
free(ListenPort);
ListenHost = oldListenHost;
ListenPort = oldListenPort;
return;
}
tcpnonblock(newlsock);
tcpnodelay(newlsock);
tcpreuseaddr(newlsock);
if (tcpsetacceptfilter(newlsock)<0 && errno!=ENOTSUP) {
mfs_errlog_silent(LOG_NOTICE,"master <-> metaloggers module: can't set accept filter");
}
if (tcpstrlisten(newlsock,ListenHost,ListenPort,100)<0) {
mfs_arg_errlog(LOG_ERR,"master <-> metaloggers module: socket address has changed, but can't listen on socket (%s:%s)",ListenHost,ListenPort);
free(ListenHost);
free(ListenPort);
ListenHost = oldListenHost;
ListenPort = oldListenPort;
tcpclose(newlsock);
return;
}
mfs_arg_syslog(LOG_NOTICE,"master <-> metaloggers module: socket address has changed, now listen on %s:%s",ListenHost,ListenPort);
free(oldListenHost);
free(oldListenPort);
tcpclose(lsock);
lsock = newlsock;
ChangelogSecondsToRemember = cfg_getuint16("MATOML_LOG_PRESERVE_SECONDS",600);
if (ChangelogSecondsToRemember>3600) {
syslog(LOG_WARNING,"Number of seconds of change logs to be preserved in master is too big (%"PRIu16") - decreasing to 3600 seconds",ChangelogSecondsToRemember);
ChangelogSecondsToRemember=3600;
}
}
void csserv_reload(void) {
char *oldListenHost,*oldListenPort;
int newlsock;
// ThreadedServer = 1-ThreadedServer;
oldListenHost = ListenHost;
oldListenPort = ListenPort;
ListenHost = cfg_getstr("CSSERV_LISTEN_HOST","*");
ListenPort = cfg_getstr("CSSERV_LISTEN_PORT",DEFAULT_CS_DATA_PORT);
if (strcmp(oldListenHost,ListenHost)==0 && strcmp(oldListenPort,ListenPort)==0) {
free(oldListenHost);
free(oldListenPort);
mfs_arg_syslog(LOG_NOTICE,"main server module: socket address hasn't changed (%s:%s)",ListenHost,ListenPort);
return;
}
newlsock = tcpsocket();
if (newlsock<0) {
mfs_errlog(LOG_WARNING,"main server module: socket address has changed, but can't create new socket");
free(ListenHost);
free(ListenPort);
ListenHost = oldListenHost;
ListenPort = oldListenPort;
return;
}
tcpnonblock(newlsock);
tcpnodelay(newlsock);
tcpreuseaddr(newlsock);
if (tcpstrlisten(newlsock,ListenHost,ListenPort,100)<0) {
mfs_arg_errlog(LOG_ERR,"main server module: socket address has changed, but can't listen on socket (%s:%s)",ListenHost,ListenPort);
free(ListenHost);
free(ListenPort);
ListenHost = oldListenHost;
ListenPort = oldListenPort;
tcpclose(newlsock);
return;
}
if (tcpsetacceptfilter(newlsock)<0 && errno!=ENOTSUP) {
mfs_errlog_silent(LOG_NOTICE,"main server module: can't set accept filter");
}
mfs_arg_syslog(LOG_NOTICE,"main server module: socket address has changed, now listen on %s:%s",ListenHost,ListenPort);
free(oldListenHost);
free(oldListenPort);
tcpclose(lsock);
lsock = newlsock;
}
int mainserv_connect(uint32_t fwdip,uint16_t fwdport,uint32_t timeout) {
int fwdsock;
fwdsock = tcpsocket();
if (fwdsock<0) {
mfs_errlog(LOG_WARNING,"create socket, error");
return -1;
}
if (tcpnonblock(fwdsock)<0) {
mfs_errlog(LOG_WARNING,"set nonblock, error");
tcpclose(fwdsock);
return -1;
}
if (tcpnumtoconnect(fwdsock,fwdip,fwdport,timeout)<0) {
mfs_arg_errlog(LOG_WARNING,"connect to %u.%u.%u.%u:%u failed, error",(fwdip>>24)&0xFF,(fwdip>>16)&0xFF,(fwdip>>8)&0xFF,fwdip&0xFF,fwdport);
tcpclose(fwdsock);
return -1;
}
int matoslaserv_init() {
ListenHost = cfg_getstr("MATOSLA_LISTEN_HOST","*");
ListenPort = cfg_getstr("MATOSLA_LISTEN_PORT","9423");
BindHost = cfg_getstr("BIND_HOST","*");
MaxReconnect = cfg_getuint32("MAX_RECONNECT",10);
sla_worker_addr[0].host = cfg_getstr("SLA_SYNC_WORKER_HOST_1","*");
sla_worker_addr[0].port = cfg_getstr("SLA_SYNC_WORKER_PORT_1","9422");
sla_worker_addr[1].host = cfg_getstr("SLA_SYNC_WORKER_HOST_2","*");
sla_worker_addr[1].port = cfg_getstr("SLA_SSYNC_WORKER_PORT_2","9422");
sla_worker_addr[2].host = cfg_getstr("SLA_SSYNC_WORKER_HOST_3","*");
sla_worker_addr[2].port = cfg_getstr("SLA_SSYNC_WORKER_PORT_3","9422");
sla_worker_addr[3].host = cfg_getstr("SLA_SSYNC_WORKER_HOST_4","*");
sla_worker_addr[3].port = cfg_getstr("SLA_SSYNC_WORKER_PORT_4","9422");
first_add_listen_sock = 0;
lsock = tcpsocket();
if (lsock<0) {
MFSLOG(LOG_ERR,"matosla: socket error: %m");
fprintf(msgfd,"master <-> metaloggers module: can't create socket\n");
return -1;
}
tcpnonblock(lsock);
tcpnodelay(lsock);
tcpreuseaddr(lsock);
if (sla_worker_addr_resolve()<0) {
MFSLOG(LOG_NOTICE,"sla_worker_addr_resolve failed");
}
if (tcpsetacceptfilter(lsock)<0) {
MFSLOG(LOG_NOTICE,"matosla: can't set accept filter: %m");
}
if (tcpstrlisten(lsock,ListenHost,ListenPort,1024)<0) {
MFSLOG(LOG_ERR,"matosla: listen error: %m");
fprintf(msgfd,"master <-> metaloggers module: can't listen on socket\n");
return -1;
}
MFSLOG(LOG_NOTICE,"matosla: listen on %s:%s",ListenHost,ListenPort);
fprintf(msgfd,"master <-> metaloggers module: listen on %s:%s\n",ListenHost,ListenPort);
matoslaservhead = NULL;
main_destructregister(matoslaserv_term);
main_epollregister(matoslaserv_desc,matoslaserv_serve);
return 0;
}
/* ning:
* 这是从 mfs matomlserv.c, matocsserv.c, matocuserv.c 三个代码里面提出来的公用代码
*/
int server_socket(char * host, char * port){
int lsock = tcpsocket();
if (lsock<0) {
logging(LOG_ERROR, "can't create socket !!!");
return -1;
}
tcpnonblock(lsock);
tcpnodelay(lsock);
tcpreuseaddr(lsock);
if (tcpsetacceptfilter(lsock)<0) {
logging(LOG_INFO,"can't set accept filter: %m"); //TODO: I do not know what this for
}
if (tcpstrlisten(lsock, host ,port,100)<0) {
logging(LOG_ERROR,"can't Listen on socket : %m");
return -1;
}
logging(LOG_INFO,"listen on %s:%s", host, port);
return lsock;
}
int serv_connect(ppfs_conn_entry* e,int ip,int port){
int fd = tcpsocket();
if(fd < 0){
fprintf(stderr,"cannot create socket\n");
return -1;
}
tcpnodelay(fd);
if(tcpnumconnect(fd,ip,port) < 0){
fprintf(stderr,"cannot connect to %X:%d\n",ip,port);
return -1;
}
e->sockfd = fd;
tcpgetpeer(fd,&(e->peerip),NULL);
//@TODO: add login
return fd;
}
//initialize listen port
int masterconn_initlisten() {
lsock = tcpsocket();
if (lsock<0) {
MFSLOG(LOG_ERR,"smtoma: socket error: %m");
fprintf(msgfd,"shadow master <-> master module: can't create socket\n");
return -1;
}
tcpnonblock(lsock);
tcpnodelay(lsock);
tcpreuseaddr(lsock);
if (tcpsetacceptfilter(lsock)<0) {
MFSLOG(LOG_NOTICE,"smtoma: can't set accept filter: %m");
}
if (tcpstrlisten(lsock,ListenHost,ListenPort,100)<0) {
MFSLOG(LOG_ERR,"smtoma: listen port:%s error: %m\n", ListenPort);
fprintf(msgfd,"shadow master to master module: can't listen on socket\n");
return -1;
}
MFSLOG(LOG_NOTICE,"smtoma: listen on %s:%s",ListenHost,ListenPort);
fprintf(msgfd,"slave <-> master module: listen on %s:%s\n",ListenHost,ListenPort);
return 0;
}
int main(int argc, char** argv) {
//*****DEALING WITH ARGUEMENTS!!
if(argc < 3){
printf("wrong number of arguments! (%i)\n", (argc-1));
exit(1);
}
int port = atoi(argv[1]);
if(port < 1 || port > 0xffff){
perror("port is invalid. i.e. > 0xffff or < 1 or not a number");
exit(1);
}
//do something with the musics
int num_stations = argc - 2;
char* stations[num_stations];
pthread_mutex_t thrmutex[num_stations];
fd_set master;
fd_set readfds;
int maxfd;
FD_ZERO(&master);
FD_ZERO(&readfds);
struct sockaddr_in temp;
struct client *root = (struct client *) malloc(sizeof(struct client));
root->cfd = -1;
memcpy(&root->sockaddr_cli, &temp, sizeof(temp));
root->station = -1;
root->udpport = -1;
root->c = NULL;
struct threadArgs thread_args[num_stations];
int i;
for(i = 0; i<num_stations; i++){
stations[i] = argv[i+2];
pthread_mutex_init(&thrmutex[i], NULL);
thread_args[i].station_number = i;
thread_args[i].num_stations = num_stations;
thread_args[i].master = &master;
thread_args[i].file = stations[i];
thread_args[i].root = root;
thread_args[i].mutex = &thrmutex[i];
}
//*****INITIATION OF THREADS FOR STATIONS!
pthread_t threads[num_stations];
for(i=0; i<num_stations; i++){
int t = pthread_create(&threads[i], NULL, station_thread, &thread_args[i]);
if(t != 0){
perror("failed to create threads for the stations!");
exit(1);
}
}
//*****SOCKET CONNECTION!
//socket part
int fd = tcpsocket(port, INADDR_ANY);
FD_SET(0, &master);
FD_SET(fd, &master);
if(listen (fd, 5) < 0){
close(fd);
perror("could not listen!");
exit(1);
}
struct sockaddr_in sin;
bzero (&sin, sizeof (sin));
sin.sin_family = AF_INET;
sin.sin_port = htons (port);
sin.sin_addr.s_addr = htonl (INADDR_ANY);
socklen_t sinlen;
struct client helloList = {-1, temp, -1, -1, NULL};
fd_set hello;
FD_ZERO(&hello);
maxfd = fd;
for(;;){
readfds = master;
if(select(maxfd+1, &readfds, NULL, NULL, NULL) < 0){
perror("select error");
exit(1);
}
int i;
for(i = 0; i <= maxfd; i++){
if(FD_ISSET(i, &readfds)){
//if the server requests from stdin
if(i == 0){
char in[1];
if(scanf("%s", in) == EOF){
write(1, "**failed to read the input.\n", sizeof("**failed to read the input.\n"));
}
//quit the server
if(in[0] == 'q'){
close(fd);
exit(1);
}
//print out the station list;
if(in[0] == 'p'){
printStations(num_stations, root);
}
}
//if new client connects to the server.
else if(i == fd){
struct client *newcli = (struct client*) malloc(sizeof(struct client));
bzero(newcli, sizeof(struct client));
sinlen = sizeof(newcli->sockaddr_cli);
int newfd = accept(fd, (struct sockaddr *) &newcli->sockaddr_cli, &sinlen);
if(newfd < 0){
free(newcli);
perror("accept error");
}else{
FD_SET(newfd, &master);
if(newfd > maxfd)
maxfd = newfd;
//add to hello list
FD_SET(newfd, &hello);
newcli->cfd = newfd;
newcli->udpport = -1;
newcli->station = -1;
newcli->c = NULL;
addToCliList(&helloList, newcli, NULL, 0);
printToSTD(i, "connection accepted. Expecting hello.\n");
}
}
//if the client has not sent a hello yet
else if(FD_ISSET(i, &hello)){
FD_CLR(i, &hello);
char hello[3];
int receive_hello = recv(i, hello, sizeof(hello), 0);
if(receive_hello < 0){
printToSTD(i, "");
perror("Failed to receive the request!");
//closing connection
FD_CLR(i, &master);
free(popCli(root, i, thrmutex, num_stations));
close(i);
printToSTD(i, "closed the connection.\n");
}else if(receive_hello == 0){
//closing connection
FD_CLR(i, &master);
free(popCli(root, i, thrmutex, num_stations));
close(i);
printToSTD(i, "the client closed the connection.\n");
}else{
if((uint8_t)hello[0] != 0){
sendInvalidMessage(i, "wrong type of request. hello expected.", &master, root, thrmutex, num_stations);
printToSTD(i, "wrong type of request requested.\n");
//closing connection
FD_CLR(i, &master);
free(popCli(root, i, thrmutex, num_stations));
close(i);
printToSTD(i, "closed the connection.\n");
}else{
uint16_t cport = (uint16_t)hello[1] << 8;
cport += (uint8_t)hello[2];
//update the client's port
struct client *cli = popCli(&helloList, i, NULL, 0);
if(cli == NULL){
printToSTD(i, "error with connection, the client could not be found.\n");
FD_CLR(i, &master);
close(i);
}else{
cli->udpport = cport;
cli->sockaddr_cli.sin_port = htons(cport);
cli->sockaddr_cli.sin_family = AF_INET;
addToCliList(root, cli, &thrmutex, num_stations);
//send welcome
char welcome[3];
welcome[0] = 0;
welcome[1] = (uint8_t)(num_stations >> 8);
welcome[2] = (uint8_t)num_stations;
send(i, welcome, sizeof(welcome), 0);
printToSTD(i, "hello received. welcome sent. expecting station.\n");
}
}
}
}
//if the client has completed the handshake
else{
char message[3];
int received = recv(i, message, sizeof(message), 0);
if(received < 0){
printToSTD(i, "");
perror("Failed to receive the request!");
//closing connection
FD_CLR(i, &master);
free(popCli(root, i, thrmutex, num_stations));
close(i);
printToSTD(i, "closed the connection.\n");
}else if(received == 0){
//closing connection
FD_CLR(i, &master);
free(popCli(root, i, thrmutex, num_stations));
close(i);
printToSTD(i, "the client closed the connection.\n");
}else{
if((uint8_t)message[0] != 1){
sendInvalidMessage(i, "wrong type of request. setstation request expected.", &master, root, thrmutex, num_stations);
printToSTD(i, "invalid commmand received. corresponding message sent.\n");
//closing connection
FD_CLR(i, &master);
free(popCli(root, i, thrmutex, num_stations));
close(i);
printToSTD(i, "closed the connection.\n");
}else{
uint16_t station = (uint16_t) message[1] << 8;
station += (uint8_t)message[2];
station = station;
//check to see if the station exists.
if(station >= num_stations){
char *out;
int size = sprintf(out, "the station %d does not exist.", station);
sendInvalidMessage(i, out, &master, root, thrmutex, num_stations);
//closing connection
FD_CLR(i, &master);
free(popCli(root, i, thrmutex, num_stations));
close(i);
printToSTD(i, "closed the connection.\n");
}else{
if(addToStation(station, stations[station], getCli(root, i),
&master, root, thrmutex, num_stations) == 0){
printToSTD(i, "the client's information could not be found in the root list.\n");
FD_CLR(i, &master);
close(i);
printToSTD(i, "closed the connection.\n");
}
}
}
}
}
}
}
/* srcs: srccnt * (chunkid:64 version:32 ip:32 port:16) */
uint8_t replicate(uint64_t chunkid,uint32_t version,uint8_t srccnt,const uint8_t *srcs) {
replication r;
uint8_t status,i,vbuffs,first;
uint16_t b,blocks;
uint32_t xcrc,crc;
uint8_t *wptr;
const uint8_t *rptr;
int s;
if (srccnt==0) {
return ERROR_EINVAL;
}
// syslog(LOG_NOTICE,"replication begin (chunkid:%08"PRIX64",version:%04"PRIX32",srccnt:%"PRIu8")",chunkid,version,srccnt);
pthread_mutex_lock(&statslock);
stats_repl++;
pthread_mutex_unlock(&statslock);
// init replication structure
r.chunkid = chunkid;
r.version = version;
r.srccnt = 0;
r.created = 0;
r.opened = 0;
r.fds = malloc(sizeof(struct pollfd)*srccnt);
passert(r.fds);
r.repsources = malloc(sizeof(repsrc)*srccnt);
passert(r.repsources);
if (srccnt>1) {
r.xorbuff = malloc(65536+4);
passert(r.xorbuff);
} else {
r.xorbuff = NULL;
}
// create chunk
status = hdd_create(chunkid,0);
if (status!=STATUS_OK) {
syslog(LOG_NOTICE,"replicator: hdd_create status: %u",status);
rep_cleanup(&r);
return status;
}
r.created = 1;
// init sources
r.srccnt = srccnt;
for (i=0 ; i<srccnt ; i++) {
r.repsources[i].chunkid = get64bit(&srcs);
r.repsources[i].version = get32bit(&srcs);
r.repsources[i].ip = get32bit(&srcs);
r.repsources[i].port = get16bit(&srcs);
r.repsources[i].sock = -1;
r.repsources[i].packet = NULL;
}
// connect
for (i=0 ; i<srccnt ; i++) {
s = tcpsocket();
if (s<0) {
mfs_errlog_silent(LOG_NOTICE,"replicator: socket error");
rep_cleanup(&r);
return ERROR_CANTCONNECT;
}
r.repsources[i].sock = s;
r.fds[i].fd = s;
if (tcpnonblock(s)<0) {
mfs_errlog_silent(LOG_NOTICE,"replicator: nonblock error");
rep_cleanup(&r);
return ERROR_CANTCONNECT;
}
s = tcpnumconnect(s,r.repsources[i].ip,r.repsources[i].port);
if (s<0) {
mfs_errlog_silent(LOG_NOTICE,"replicator: connect error");
rep_cleanup(&r);
return ERROR_CANTCONNECT;
}
if (s==0) {
r.repsources[i].mode = IDLE;
} else {
r.repsources[i].mode = CONNECTING;
}
}
if (rep_wait_for_connection(&r,CONNMSECTO)<0) {
rep_cleanup(&r);
return ERROR_CANTCONNECT;
}
// open chunk
status = hdd_open(chunkid);
if (status!=STATUS_OK) {
syslog(LOG_NOTICE,"replicator: hdd_open status: %u",status);
rep_cleanup(&r);
return status;
}
r.opened = 1;
// get block numbers
for (i=0 ; i<srccnt ; i++) {
wptr = rep_create_packet(r.repsources+i,CSTOCS_GET_CHUNK_BLOCKS,8+4);
if (wptr==NULL) {
syslog(LOG_NOTICE,"replicator: out of memory");
rep_cleanup(&r);
return ERROR_OUTOFMEMORY;
}
put64bit(&wptr,r.repsources[i].chunkid);
put32bit(&wptr,r.repsources[i].version);
}
// send packet
if (rep_send_all_packets(&r,SENDMSECTO)<0) {
rep_cleanup(&r);
return ERROR_DISCONNECTED;
}
// receive answers
for (i=0 ; i<srccnt ; i++) {
r.repsources[i].mode = HEADER;
r.repsources[i].startptr = r.repsources[i].hdrbuff;
r.repsources[i].bytesleft = 8;
}
if (rep_receive_all_packets(&r,RECVMSECTO)<0) {
rep_cleanup(&r);
return ERROR_DISCONNECTED;
}
// get block no
blocks = 0;
for (i=0 ; i<srccnt ; i++) {
uint32_t type,size;
uint64_t pchid;
uint32_t pver;
uint16_t pblocks;
uint8_t pstatus;
rptr = r.repsources[i].hdrbuff;
type = get32bit(&rptr);
size = get32bit(&rptr);
rptr = r.repsources[i].packet;
if (rptr==NULL || type!=CSTOCS_GET_CHUNK_BLOCKS_STATUS || size!=15) {
syslog(LOG_WARNING,"replicator: got wrong answer (type/size) from (%08"PRIX32":%04"PRIX16")",r.repsources[i].ip,r.repsources[i].port);
rep_cleanup(&r);
return ERROR_DISCONNECTED;
}
pchid = get64bit(&rptr);
pver = get32bit(&rptr);
pblocks = get16bit(&rptr);
pstatus = get8bit(&rptr);
if (pchid!=r.repsources[i].chunkid) {
syslog(LOG_WARNING,"replicator: got wrong answer (chunk_status:chunkid:%"PRIX64"/%"PRIX64") from (%08"PRIX32":%04"PRIX16")",pchid,r.repsources[i].chunkid,r.repsources[i].ip,r.repsources[i].port);
rep_cleanup(&r);
return ERROR_WRONGCHUNKID;
}
if (pver!=r.repsources[i].version) {
syslog(LOG_WARNING,"replicator: got wrong answer (chunk_status:version:%"PRIX32"/%"PRIX32") from (%08"PRIX32":%04"PRIX16")",pver,r.repsources[i].version,r.repsources[i].ip,r.repsources[i].port);
rep_cleanup(&r);
return ERROR_WRONGVERSION;
}
if (pstatus!=STATUS_OK) {
syslog(LOG_NOTICE,"replicator: got status: %u from (%08"PRIX32":%04"PRIX16")",pstatus,r.repsources[i].ip,r.repsources[i].port);
rep_cleanup(&r);
return pstatus;
}
r.repsources[i].blocks = pblocks;
if (pblocks>blocks) {
blocks=pblocks;
}
}
// create read request
for (i=0 ; i<srccnt ; i++) {
if (r.repsources[i].blocks>0) {
uint32_t leng;
wptr = rep_create_packet(r.repsources+i,CUTOCS_READ,8+4+4+4);
if (wptr==NULL) {
syslog(LOG_NOTICE,"replicator: out of memory");
rep_cleanup(&r);
return ERROR_OUTOFMEMORY;
}
leng = r.repsources[i].blocks*0x10000;
put64bit(&wptr,r.repsources[i].chunkid);
put32bit(&wptr,r.repsources[i].version);
put32bit(&wptr,0);
put32bit(&wptr,leng);
} else {
rep_no_packet(r.repsources+i);
}
}
// send read request
if (rep_send_all_packets(&r,SENDMSECTO)<0) {
rep_cleanup(&r);
return ERROR_DISCONNECTED;
}
// receive data and write to hdd
for (b=0 ; b<blocks ; b++) {
// prepare receive
for (i=0 ; i<srccnt ; i++) {
if (b<r.repsources[i].blocks) {
r.repsources[i].mode = HEADER;
r.repsources[i].startptr = r.repsources[i].hdrbuff;
r.repsources[i].bytesleft = 8;
} else {
r.repsources[i].mode = IDLE;
r.repsources[i].bytesleft = 0;
}
}
// receive data
if (rep_receive_all_packets(&r,RECVMSECTO)<0) {
rep_cleanup(&r);
return ERROR_DISCONNECTED;
}
// check packets
vbuffs = 0;
for (i=0 ; i<srccnt ; i++) {
if (r.repsources[i].mode!=IDLE) {
uint32_t type,size;
uint64_t pchid;
uint16_t pblocknum;
uint16_t poffset;
uint32_t psize;
uint8_t pstatus;
rptr = r.repsources[i].hdrbuff;
type = get32bit(&rptr);
size = get32bit(&rptr);
rptr = r.repsources[i].packet;
if (rptr==NULL) {
rep_cleanup(&r);
return ERROR_DISCONNECTED;
}
if (type==CSTOCU_READ_STATUS && size==9) {
pchid = get64bit(&rptr);
pstatus = get8bit(&rptr);
rep_cleanup(&r);
if (pchid!=r.repsources[i].chunkid) {
syslog(LOG_WARNING,"replicator: got wrong answer (read_status:chunkid:%"PRIX64"/%"PRIX64") from (%08"PRIX32":%04"PRIX16")",pchid,r.repsources[i].chunkid,r.repsources[i].ip,r.repsources[i].port);
return ERROR_WRONGCHUNKID;
}
if (pstatus==STATUS_OK) { // got status too early or got incorrect packet
syslog(LOG_WARNING,"replicator: got unexpected ok status from (%08"PRIX32":%04"PRIX16")",r.repsources[i].ip,r.repsources[i].port);
return ERROR_DISCONNECTED;
}
syslog(LOG_NOTICE,"replicator: got status: %u from (%08"PRIX32":%04"PRIX16")",pstatus,r.repsources[i].ip,r.repsources[i].port);
return pstatus;
} else if (type==CSTOCU_READ_DATA && size==20+65536) {
pchid = get64bit(&rptr);
pblocknum = get16bit(&rptr);
poffset = get16bit(&rptr);
psize = get32bit(&rptr);
if (pchid!=r.repsources[i].chunkid) {
syslog(LOG_WARNING,"replicator: got wrong answer (read_data:chunkid:%"PRIX64"/%"PRIX64") from (%08"PRIX32":%04"PRIX16")",pchid,r.repsources[i].chunkid,r.repsources[i].ip,r.repsources[i].port);
rep_cleanup(&r);
return ERROR_WRONGCHUNKID;
}
if (pblocknum!=b) {
syslog(LOG_WARNING,"replicator: got wrong answer (read_data:blocknum:%"PRIu16"/%"PRIu16") from (%08"PRIX32":%04"PRIX16")",pblocknum,b,r.repsources[i].ip,r.repsources[i].port);
rep_cleanup(&r);
return ERROR_DISCONNECTED;
}
if (poffset!=0) {
syslog(LOG_WARNING,"replicator: got wrong answer (read_data:offset:%"PRIu16") from (%08"PRIX32":%04"PRIX16")",poffset,r.repsources[i].ip,r.repsources[i].port);
rep_cleanup(&r);
return ERROR_WRONGOFFSET;
}
if (psize!=65536) {
syslog(LOG_WARNING,"replicator: got wrong answer (read_data:size:%"PRIu32") from (%08"PRIX32":%04"PRIX16")",psize,r.repsources[i].ip,r.repsources[i].port);
rep_cleanup(&r);
return ERROR_WRONGSIZE;
}
} else {
syslog(LOG_WARNING,"replicator: got wrong answer (type/size) from (%08"PRIX32":%04"PRIX16")",r.repsources[i].ip,r.repsources[i].port);
rep_cleanup(&r);
return ERROR_DISCONNECTED;
}
vbuffs++;
}
}
// write data
if (vbuffs==0) { // no buffers ? - it should never happen
syslog(LOG_WARNING,"replicator: no data received for block: %"PRIu16,b);
rep_cleanup(&r);
return ERROR_DISCONNECTED;
} else if (vbuffs==1) { // xor not needed, so just find block and write it
for (i=0 ; i<srccnt ; i++) {
if (r.repsources[i].mode!=IDLE) {
rptr = r.repsources[i].packet;
status = hdd_write(chunkid,0,b,rptr+20,0,65536,rptr+16);
if (status!=STATUS_OK) {
syslog(LOG_WARNING,"replicator: write status: %u",status);
rep_cleanup(&r);
return status;
}
}
}
} else {
first=1;
if (vbuffs&1) {
xcrc = 0;
} else {
xcrc = 0xD7978EEBU; // = mycrc32_zeroblock(0,0x10000);
}
for (i=0 ; i<srccnt ; i++) {
if (r.repsources[i].mode!=IDLE) {
rptr = r.repsources[i].packet;
rptr+=16; // skip chunkid,blockno,offset and size
if (first) {
memcpy(r.xorbuff+4,rptr+4,65536);
first=0;
} else {
xordata(r.xorbuff+4,rptr+4,65536);
}
crc = get32bit(&rptr);
if (crc!=mycrc32(0,rptr,65536)) {
syslog(LOG_WARNING,"replicator: received data with wrong checksum from (%08"PRIX32":%04"PRIX16")",r.repsources[i].ip,r.repsources[i].port);
rep_cleanup(&r);
return ERROR_CRC;
}
xcrc^=crc;
}
}
wptr = r.xorbuff;
put32bit(&wptr,xcrc);
status = hdd_write(chunkid,0,b,r.xorbuff+4,0,65536,r.xorbuff);
if (status!=STATUS_OK) {
syslog(LOG_WARNING,"replicator: xor write status: %u",status);
rep_cleanup(&r);
return status;
}
}
}
// receive status
for (i=0 ; i<srccnt ; i++) {
if (r.repsources[i].blocks>0) {
// if (r.repsources[i].packet) {
// free(r.repsources[i].packet);
// r.repsources[i].packet=NULL;
// }
r.repsources[i].mode = HEADER;
r.repsources[i].startptr = r.repsources[i].hdrbuff;
r.repsources[i].bytesleft = 8;
} else {
r.repsources[i].mode = IDLE;
r.repsources[i].bytesleft = 0;
}
}
if (rep_receive_all_packets(&r,RECVMSECTO)<0) {
rep_cleanup(&r);
return ERROR_DISCONNECTED;
}
for (i=0 ; i<srccnt ; i++) {
if (r.repsources[i].blocks>0) {
uint32_t type,size;
uint64_t pchid;
uint8_t pstatus;
rptr = r.repsources[i].hdrbuff;
type = get32bit(&rptr);
size = get32bit(&rptr);
rptr = r.repsources[i].packet;
if (rptr==NULL || type!=CSTOCU_READ_STATUS || size!=9) {
syslog(LOG_WARNING,"replicator: got wrong answer (type/size) from (%08"PRIX32":%04"PRIX16")",r.repsources[i].ip,r.repsources[i].port);
rep_cleanup(&r);
return ERROR_DISCONNECTED;
}
pchid = get64bit(&rptr);
pstatus = get8bit(&rptr);
if (pchid!=r.repsources[i].chunkid) {
syslog(LOG_WARNING,"replicator: got wrong answer (read_status:chunkid:%"PRIX64"/%"PRIX64") from (%08"PRIX32":%04"PRIX16")",pchid,r.repsources[i].chunkid,r.repsources[i].ip,r.repsources[i].port);
rep_cleanup(&r);
return ERROR_WRONGCHUNKID;
}
if (pstatus!=STATUS_OK) {
syslog(LOG_NOTICE,"replicator: got status: %u from (%08"PRIX32":%04"PRIX16")",pstatus,r.repsources[i].ip,r.repsources[i].port);
rep_cleanup(&r);
return pstatus;
}
}
}
// close chunk and change version
status = hdd_close(chunkid);
if (status!=STATUS_OK) {
syslog(LOG_NOTICE,"replicator: hdd_close status: %u",status);
rep_cleanup(&r);
return status;
}
r.opened = 0;
status = hdd_version(chunkid,0,version);
if (status!=STATUS_OK) {
syslog(LOG_NOTICE,"replicator: hdd_version status: %u",status);
rep_cleanup(&r);
return status;
}
r.created = 0;
rep_cleanup(&r);
return STATUS_OK;
}
//init the thread's socket
static int sla_sync_socket_init(serventry *eptr,pthread_t tid) {
int status;
uint32_t shadow_ip = 0,bind_ip = 0,bip = 0;
uint16_t shadow_port;
int num;
int count = 0;
num = sla_worker_thread_indent(tid);
if (tcpresolve(sla_worker_addr[num].host,sla_worker_addr[num].port,&shadow_ip,&shadow_port,0)>=0) {
eptr->masterip = shadow_ip;
eptr->masterport = shadow_port;
} else {
MFSLOG(LOG_ERR,"resolve shadow's addr failed");
return -1;
}
eptr->sock = tcpsocket();
if (eptr->sock < 0) {
MFSLOG(LOG_ERR,"socket init failed");
return -1;
}
if (tcpnonblock(eptr->sock)<0) {
MFSLOG(LOG_WARNING,"set nonblock, error: %m");
tcpclose(eptr->sock);
eptr->sock=-1;
return -1;
}
tcpgetpeer(eptr->sock,&(eptr->servip),NULL);
eptr->servstrip = matoslaserv_makestrip(eptr->servip);
if (tcpresolve(BindHost,NULL,&bind_ip,NULL,1)>=0) {
eptr->bindip = bip;
} else {
eptr->bindip = 0;
}
if (eptr->bindip>0) {
if (tcpnumbind(eptr->sock,eptr->bindip,0)<0) {
MFSLOG(LOG_WARNING,"can't bind socket to given ip: %m");
tcpclose(eptr->sock);
eptr->sock=-1;
return -1;
}
}
eptr->cur_file = (file_info *)malloc(sizeof(file_info));
if (eptr->cur_file == NULL) {
MFSLOG(LOG_ERR,"FATAL,malloc file_info failed");
} else {
eptr->cur_file->fd = NULL;
eptr->cur_file->idx = 0;
}
while(count < MaxConnect) {
status = tcpnumconnect(eptr->sock,eptr->masterip,eptr->masterport);
if (status<0) {
MFSLOG(LOG_WARNING,"connect failed, error: %m (1)");
tcpclose(eptr->sock);
eptr->sock=-1;
return -1;
}
if (status==0) {
MFSLOG(LOG_NOTICE,"connected to slave");
tcpnodelay(eptr->sock);
eptr->mode=HEADER;
eptr->inputpacket.next = NULL;
eptr->inputpacket.bytesleft = 8;
eptr->inputpacket.startptr = eptr->hdrbuff;
eptr->inputpacket.packet = NULL;
eptr->outputhead = NULL;
eptr->outputtail = &(eptr->outputhead);
eptr->lastread = eptr->lastwrite = get_current_time();
count = 100;
break;
} else {
MFSLOG(LOG_ERR,"connect failed, error: %m (2)");
}
count++;
sleep(1);
}
return 0;
}
/* main working thread | glock:UNLOCKED */
void* write_worker(void *arg) {
uint32_t z1,z2,z3;
uint8_t *data;
int fd;
int i;
struct pollfd pfd[2];
uint32_t sent,rcvd;
uint8_t recvbuff[21];
uint8_t sendbuff[32];
#ifdef HAVE_WRITEV
struct iovec siov[2];
#endif
uint8_t pipebuff[1024];
uint8_t *wptr;
const uint8_t *rptr;
uint32_t reccmd;
uint32_t recleng;
uint64_t recchunkid;
uint32_t recwriteid;
uint8_t recstatus;
#ifdef WORKER_DEBUG
uint32_t partialblocks;
uint32_t bytessent;
char debugchain[200];
uint32_t cl;
#endif
const uint8_t *cp,*cpe;
uint32_t chainip[10];
uint16_t chainport[10];
uint16_t chainelements;
uint16_t chindx;
uint32_t ip;
uint16_t port;
uint32_t srcip;
uint64_t mfleng;
uint64_t maxwroffset;
uint64_t chunkid;
uint32_t version;
uint32_t nextwriteid;
const uint8_t *chain;
uint32_t chainsize;
const uint8_t *csdata;
uint32_t csdatasize;
uint8_t westatus;
uint8_t wrstatus;
int status;
uint8_t waitforstatus;
uint8_t havedata;
struct timeval start,now,lastrcvd,lrdiff;
uint8_t cnt;
inodedata *id;
cblock *cb,*rcb;
// inodedata *id;
chainelements = 0;
(void)arg;
for (;;) {
for (cnt=0 ; cnt<chainelements ; cnt++) {
csdb_writedec(chainip[cnt],chainport[cnt]);
}
chainelements=0;
// get next job
queue_get(jqueue,&z1,&z2,&data,&z3);
id = (inodedata*)data;
pthread_mutex_lock(&glock);
if (id->datachainhead) {
chindx = id->datachainhead->chindx;
} else {
syslog(LOG_WARNING,"writeworker got inode with no data to write !!!");
chindx = 0xFFFF;
status = EINVAL; // this should never happen, so status is not important - just anything
}
status = id->status;
pthread_mutex_unlock(&glock);
if (status) {
write_job_end(id,status,0);
continue;
}
// syslog(LOG_NOTICE,"file: %"PRIu32", index: %"PRIu16" - debug1",id->inode,chindx);
// get chunk data from master
wrstatus = fs_writechunk(id->inode,chindx,&mfleng,&chunkid,&version,&csdata,&csdatasize);
if (wrstatus!=STATUS_OK) {
syslog(LOG_WARNING,"file: %"PRIu32", index: %"PRIu16" - fs_writechunk returns status %d",id->inode,chindx,wrstatus);
if (wrstatus!=ERROR_LOCKED) {
if (wrstatus==ERROR_ENOENT) {
write_job_end(id,EBADF,0);
} else if (wrstatus==ERROR_QUOTA) {
write_job_end(id,EDQUOT,0);
} else if (wrstatus==ERROR_NOSPACE) {
write_job_end(id,ENOSPC,0);
} else {
id->trycnt++;
if (id->trycnt>=maxretries) {
if (wrstatus==ERROR_NOCHUNKSERVERS) {
write_job_end(id,ENOSPC,0);
} else {
write_job_end(id,EIO,0);
}
} else {
write_delayed_enqueue(id,1+(id->trycnt<30)?(id->trycnt/3):10);
}
}
} else {
write_delayed_enqueue(id,1+(id->trycnt<30)?(id->trycnt/3):10);
}
continue; // get next job
}
if (csdata==NULL || csdatasize==0) {
syslog(LOG_WARNING,"file: %"PRIu32", index: %"PRIu16", chunk: %"PRIu64", version: %"PRIu32" - there are no valid copies",id->inode,chindx,chunkid,version);
id->trycnt+=6;
if (id->trycnt>=maxretries) {
write_job_end(id,ENXIO,0);
} else {
write_delayed_enqueue(id,60);
}
continue;
}
cp = csdata;
cpe = csdata+csdatasize;
while (cp<cpe && chainelements<10) {
chainip[chainelements] = get32bit(&cp);
chainport[chainelements] = get16bit(&cp);
csdb_writeinc(chainip[chainelements],chainport[chainelements]);
chainelements++;
}
chain = csdata;
ip = get32bit(&chain);
port = get16bit(&chain);
chainsize = csdatasize-6;
gettimeofday(&start,NULL);
/*
if (csdatasize>CSDATARESERVE) {
csdatasize = CSDATARESERVE;
}
memcpy(wrec->csdata,csdata,csdatasize);
wrec->csdatasize=csdatasize;
while (csdatasize>=6) {
tmpip = get32bit(&csdata);
tmpport = get16bit(&csdata);
csdatasize-=6;
csdb_writeinc(tmpip,tmpport);
}
*/
// make connection to cs
srcip = fs_getsrcip();
cnt=5;
while (cnt>0) {
fd = tcpsocket();
if (fd<0) {
syslog(LOG_WARNING,"can't create tcp socket: %m");
cnt=0;
}
if (srcip) {
if (tcpnumbind(fd,srcip,0)<0) {
syslog(LOG_WARNING,"can't bind socket to given ip: %m");
tcpclose(fd);
fd=-1;
break;
}
}
if (tcpnumtoconnect(fd,ip,port,200)<0) {
cnt--;
if (cnt==0) {
syslog(LOG_WARNING,"can't connect to (%08"PRIX32":%"PRIu16"): %m",ip,port);
}
tcpclose(fd);
fd=-1;
} else {
cnt=0;
}
}
if (fd<0) {
fs_writeend(chunkid,id->inode,0);
id->trycnt++;
if (id->trycnt>=maxretries) {
write_job_end(id,EIO,0);
} else {
write_delayed_enqueue(id,1+(id->trycnt<30)?(id->trycnt/3):10);
}
continue;
}
if (tcpnodelay(fd)<0) {
syslog(LOG_WARNING,"can't set TCP_NODELAY: %m");
}
#ifdef WORKER_DEBUG
partialblocks=0;
bytessent=0;
#endif
nextwriteid=1;
pfd[0].fd = fd;
pfd[1].fd = id->pipe[0];
rcvd = 0;
sent = 0;
waitforstatus=1;
havedata=1;
wptr = sendbuff;
put32bit(&wptr,CUTOCS_WRITE);
put32bit(&wptr,12+chainsize);
put64bit(&wptr,chunkid);
put32bit(&wptr,version);
// debug: syslog(LOG_NOTICE,"writeworker: init packet prepared");
cb = NULL;
status = 0;
wrstatus = STATUS_OK;
lastrcvd.tv_sec = 0;
do {
gettimeofday(&now,NULL);
if (lastrcvd.tv_sec==0) {
lastrcvd = now;
} else {
lrdiff = now;
if (lrdiff.tv_usec<lastrcvd.tv_usec) {
lrdiff.tv_sec--;
lrdiff.tv_usec+=1000000;
}
lrdiff.tv_sec -= lastrcvd.tv_sec;
lrdiff.tv_usec -= lastrcvd.tv_usec;
if (lrdiff.tv_sec>=2) {
syslog(LOG_WARNING,"file: %"PRIu32", index: %"PRIu16", chunk: %"PRIu64", version: %"PRIu32" - writeworker: connection with (%08"PRIX32":%"PRIu16") was timed out (unfinished writes: %"PRIu8"; try counter: %"PRIu32")",id->inode,chindx,chunkid,version,ip,port,waitforstatus,id->trycnt+1);
break;
}
}
if (now.tv_usec<start.tv_usec) {
now.tv_sec--;
now.tv_usec+=1000000;
}
now.tv_sec -= start.tv_sec;
now.tv_usec -= start.tv_usec;
if (havedata==0 && now.tv_sec<5 && waitforstatus<5) {
pthread_mutex_lock(&glock);
if (cb==NULL) {
if (id->datachainhead) {
if (id->datachainhead->to-id->datachainhead->from==65536 || waitforstatus<=1) {
cb = id->datachainhead;
havedata=1;
}
}
} else {
if (cb->next) {
if (cb->next->chindx==chindx) {
if (cb->next->to-cb->next->from==65536 || waitforstatus<=1) {
cb = cb->next;
havedata=1;
}
}
} else {
id->waitingworker=1;
}
}
if (havedata==1) {
cb->writeid = nextwriteid++;
// debug: syslog(LOG_NOTICE,"writeworker: data packet prepared (writeid:%"PRIu32",pos:%"PRIu16")",cb->writeid,cb->pos);
waitforstatus++;
wptr = sendbuff;
put32bit(&wptr,CUTOCS_WRITE_DATA);
put32bit(&wptr,24+(cb->to-cb->from));
put64bit(&wptr,chunkid);
put32bit(&wptr,cb->writeid);
put16bit(&wptr,cb->pos);
put16bit(&wptr,cb->from);
put32bit(&wptr,cb->to-cb->from);
put32bit(&wptr,mycrc32(0,cb->data+cb->from,cb->to-cb->from));
#ifdef WORKER_DEBUG
if (cb->to-cb->from<65536) {
partialblocks++;
}
bytessent+=(cb->to-cb->from);
#endif
sent=0;
}
pthread_mutex_unlock(&glock);
}
pfd[0].events = POLLIN | (havedata?POLLOUT:0);
pfd[0].revents = 0;
pfd[1].events = POLLIN;
pfd[1].revents = 0;
if (poll(pfd,2,100)<0) { /* correct timeout - in msec */
syslog(LOG_WARNING,"writeworker: poll error: %m");
status=EIO;
break;
}
if (pfd[1].revents&POLLIN) { // used just to break poll - so just read all data from pipe to empty it
i = read(id->pipe[0],pipebuff,1024);
}
if (pfd[0].revents&POLLIN) {
i = read(fd,recvbuff+rcvd,21-rcvd);
if (i==0) { // connection reset by peer ,读取文件头错误
syslog(LOG_WARNING,"file: %"PRIu32", index: %"PRIu16", chunk: %"PRIu64", version: %"PRIu32" - writeworker: connection with (%08"PRIX32":%"PRIu16") was reset by peer (unfinished writes: %"PRIu8"; try counter: %"PRIu32")",id->inode,chindx,chunkid,version,ip,port,waitforstatus,id->trycnt+1);
status=EIO;
break;
}
gettimeofday(&lastrcvd,NULL);
rcvd+=i;
if (rcvd==21) {
rptr = recvbuff;
reccmd = get32bit(&rptr);
recleng = get32bit(&rptr);
recchunkid = get64bit(&rptr);
recwriteid = get32bit(&rptr);
recstatus = get8bit(&rptr);
if (reccmd!=CSTOCU_WRITE_STATUS || recleng!=13) {
syslog(LOG_WARNING,"writeworker: got unrecognized packet from chunkserver (cmd:%"PRIu32",leng:%"PRIu32")",reccmd,recleng);
status=EIO;
break;
}
if (recchunkid!=chunkid) {
syslog(LOG_WARNING,"writeworker: got unexpected packet (expected chunkdid:%"PRIu64",packet chunkid:%"PRIu64")",chunkid,recchunkid);
status=EIO;
break;
}
if (recstatus!=STATUS_OK) {
syslog(LOG_WARNING,"writeworker: write error: %"PRIu8,recstatus);
wrstatus=recstatus;
break;
}
// debug: syslog(LOG_NOTICE,"writeworker: received status ok for writeid:%"PRIu32,recwriteid);
if (recwriteid>0) {
pthread_mutex_lock(&glock);
for (rcb = id->datachainhead ; rcb && rcb->writeid!=recwriteid ; rcb=rcb->next) {}
if (rcb==NULL) {
syslog(LOG_WARNING,"writeworker: got unexpected status (writeid:%"PRIu32")",recwriteid);
pthread_mutex_unlock(&glock);
status=EIO;
break;
}
if (rcb==cb) { // current block,cb为当前块儿指针
// debug: syslog(LOG_NOTICE,"writeworker: received status for current block");
if (havedata) { // got status ok before all data had been sent - error
syslog(LOG_WARNING,"writeworker: got status OK before all data have been sent");
pthread_mutex_unlock(&glock);
status=EIO;
break;
} else {
cb = NULL;
}
}
if (rcb->prev) {//将rcb所指块儿从链表中取出
rcb->prev->next = rcb->next;
} else {
id->datachainhead = rcb->next;
}
if (rcb->next) {
rcb->next->prev = rcb->prev;
} else {
id->datachaintail = rcb->prev;
}
maxwroffset = (((uint64_t)(chindx))<<26)+(((uint32_t)(rcb->pos))<<16)+rcb->to;
if (maxwroffset>mfleng) {
mfleng=maxwroffset;
}
write_cb_release(rcb);//
id->cacheblocks--;
if (id->cachewaiting>0) {
pthread_cond_broadcast(&(id->cachecond));
}
pthread_mutex_unlock(&glock);
}
waitforstatus--;
rcvd=0;
}
}
if (havedata && (pfd[0].revents&POLLOUT)) {
if (cb==NULL) { // havedata==1 && cb==NULL means sending first packet (CUTOCS_WRITE)
if (sent<20) {
#ifdef HAVE_WRITEV //将多个数据存储在一起，将驻留在两个或更多的不连接的缓冲区中的数据一次写出去
if (chainsize>0) {
siov[0].iov_base = sendbuff+sent;
siov[0].iov_len = 20-sent;
siov[1].iov_base = (char*)chain; // discard const (safe - because it's used in writev)
siov[1].iov_len = chainsize;
i = writev(fd,siov,2);
} else {
#endif
i = write(fd,sendbuff+sent,20-sent);
#ifdef HAVE_WRITEV
}
#endif
} else {
i = write(fd,chain+(sent-20),chainsize-(sent-20));
}
if (i<0) {
syslog(LOG_WARNING,"file: %"PRIu32", index: %"PRIu16", chunk: %"PRIu64", version: %"PRIu32" - writeworker: connection with (%08"PRIX32":%"PRIu16") was reset by peer (unfinished writes: %"PRIu8"; try counter: %"PRIu32")",id->inode,chindx,chunkid,version,ip,port,waitforstatus,id->trycnt+1);
status=EIO;
break;
}
sent+=i;
if (sent==20+chainsize) {
havedata=0;
}
} else {
if (sent<32) {
#ifdef HAVE_WRITEV
siov[0].iov_base = sendbuff+sent;
siov[0].iov_len = 32-sent;
siov[1].iov_base = cb->data+cb->from;
siov[1].iov_len = cb->to-cb->from;
i = writev(fd,siov,2);
#else
i = write(fd,sendbuff+sent,32-sent);
#endif
} else {
i = write(fd,cb->data+cb->from+(sent-32),cb->to-cb->from-(sent-32));
}
if (i<0) {
syslog(LOG_WARNING,"file: %"PRIu32", index: %"PRIu16", chunk: %"PRIu64", version: %"PRIu32" - writeworker: connection with (%08"PRIX32":%"PRIu16") was reset by peer (unfinished writes: %"PRIu8"; try counter: %"PRIu32")",id->inode,chindx,chunkid,version,ip,port,waitforstatus,id->trycnt+1);
status=EIO;
break;
}
sent+=i;
if (sent==32+cb->to-cb->from) {
havedata=0;
}
}
}
} while (waitforstatus>0 && now.tv_sec<10);////////////////////
id->waitingworker=0;
tcpclose(fd);
#ifdef WORKER_DEBUG
gettimeofday(&now,NULL);
if (now.tv_usec<start.tv_usec) {
now.tv_sec--;
now.tv_usec+=1000000;
}
now.tv_sec -= start.tv_sec;
now.tv_usec -= start.tv_usec;
cl=0;
for (cnt=0 ; cnt<chainelements ; cnt++) {
cl+=snprintf(debugchain+cl,200-cl,"%u.%u.%u.%u:%u->",(chainip[cnt]>>24)&255,(chainip[cnt]>>16)&255,(chainip[cnt]>>8)&255,chainip[cnt]&255,chainport[cnt]);
}
if (cl>=2) {
debugchain[cl-2]='\0';
}
syslog(LOG_NOTICE,"worker %lu sent %"PRIu32" blocks (%"PRIu32" partial) of chunk %016"PRIX64"_%08"PRIX32", received status for %"PRIu32" blocks (%"PRIu32" lost), bw: %.6lfMB ( %"PRIu32" B / %.0lf us ), chain: %s",(unsigned long)arg,nextwriteid-1,partialblocks,chunkid,version,nextwriteid-1-waitforstatus,waitforstatus,(double)bytessent/((double)(now.tv_sec)*1000000+(double)(now.tv_usec)),bytessent,((double)(now.tv_sec)*1000000+(double)(now.tv_usec)),debugchain);
#endif
for (cnt=0 ; cnt<10 ; cnt++) {
westatus = fs_writeend(chunkid,id->inode,mfleng);
if (westatus!=STATUS_OK) {
usleep(100000+(10000<<cnt));
} else {
break;
}
}
if (westatus!=STATUS_OK) {
write_job_end(id,ENXIO,0);
} else if (status!=0 || wrstatus!=STATUS_OK) {
if (wrstatus!=STATUS_OK) { // convert MFS status to OS errno
if (wrstatus==ERROR_NOSPACE) {
status=ENOSPC;
} else {
status=EIO;
}
}
id->trycnt++;
if (id->trycnt>=maxretries) {
write_job_end(id,status,0);
} else {
write_job_end(id,0,1+(id->trycnt<30)?(id->trycnt/3):10);
}
} else {
read_inode_ops(id->inode);
write_job_end(id,0,0);
}
}
}
/* srcs: srccnt * (chunkid:64 version:32 ip:32 port:16) */
uint8_t replicate(uint64_t chunkid,uint32_t version,const uint32_t xormasks[4],uint8_t srccnt,const uint8_t *srcs) {
replication r;
uint8_t status,i,j,vbuffs,first;
uint16_t b,blocks;
uint32_t xcrc[4],crc;
uint32_t codeindex,codeword;
uint8_t *wptr;
const uint8_t *rptr;
int s;
if (srccnt==0) {
return ERROR_EINVAL;
}
// syslog(LOG_NOTICE,"replication begin (chunkid:%08"PRIX64",version:%04"PRIX32",srccnt:%"PRIu8")",chunkid,version,srccnt);
pthread_mutex_lock(&statslock);
stats_repl++;
pthread_mutex_unlock(&statslock);
// init replication structure
r.chunkid = chunkid;
r.version = version;
r.srccnt = 0;
r.created = 0;
r.opened = 0;
r.fds = malloc(sizeof(struct pollfd)*srccnt);
passert(r.fds);
r.repsources = malloc(sizeof(repsrc)*srccnt);
passert(r.repsources);
if (srccnt>1) {
r.xorbuff = malloc(MFSBLOCKSIZE+4);
passert(r.xorbuff);
} else {
r.xorbuff = NULL;
}
// create chunk
status = hdd_create(chunkid,0);
if (status!=STATUS_OK) {
syslog(LOG_NOTICE,"replicator: hdd_create status: %s",mfsstrerr(status));
rep_cleanup(&r);
return status;
}
r.created = 1;
// init sources
r.srccnt = srccnt;
for (i=0 ; i<srccnt ; i++) {
r.repsources[i].chunkid = get64bit(&srcs);
r.repsources[i].version = get32bit(&srcs);
r.repsources[i].ip = get32bit(&srcs);
r.repsources[i].port = get16bit(&srcs);
r.repsources[i].sock = -1;
r.repsources[i].packet = NULL;
}
// connect
for (i=0 ; i<srccnt ; i++) {
s = tcpsocket();
if (s<0) {
mfs_errlog_silent(LOG_NOTICE,"replicator: socket error");
rep_cleanup(&r);
return ERROR_CANTCONNECT;
}
r.repsources[i].sock = s;
r.fds[i].fd = s;
if (tcpnonblock(s)<0) {
mfs_errlog_silent(LOG_NOTICE,"replicator: nonblock error");
rep_cleanup(&r);
return ERROR_CANTCONNECT;
}
s = tcpnumconnect(s,r.repsources[i].ip,r.repsources[i].port);
if (s<0) {
mfs_errlog_silent(LOG_NOTICE,"replicator: connect error");
rep_cleanup(&r);
return ERROR_CANTCONNECT;
}
if (s==0) {
r.repsources[i].mode = IDLE;
} else {
r.repsources[i].mode = CONNECTING;
}
}
if (rep_wait_for_connection(&r,CONNMSECTO)<0) {
rep_cleanup(&r);
return ERROR_CANTCONNECT;
}
// disable Nagle
for (i=0 ; i<srccnt ; i++) {
tcpnodelay(r.repsources[i].sock);
}
// open chunk
status = hdd_open(chunkid,0);
if (status!=STATUS_OK) {
syslog(LOG_NOTICE,"replicator: hdd_open status: %s",mfsstrerr(status));
rep_cleanup(&r);
return status;
}
r.opened = 1;
// get block numbers
for (i=0 ; i<srccnt ; i++) {
wptr = rep_create_packet(r.repsources+i,ANTOCS_GET_CHUNK_BLOCKS,8+4);
if (wptr==NULL) {
syslog(LOG_NOTICE,"replicator: out of memory");
rep_cleanup(&r);
return ERROR_OUTOFMEMORY;
}
put64bit(&wptr,r.repsources[i].chunkid);
put32bit(&wptr,r.repsources[i].version);
}
// send packet
if (rep_send_all_packets(&r,SENDMSECTO)<0) {
rep_cleanup(&r);
return ERROR_DISCONNECTED;
}
// receive answers
for (i=0 ; i<srccnt ; i++) {
r.repsources[i].mode = HEADER;
r.repsources[i].startptr = r.repsources[i].hdrbuff;
r.repsources[i].bytesleft = 8;
}
if (rep_receive_all_packets(&r,RECVMSECTO)<0) {
rep_cleanup(&r);
return ERROR_DISCONNECTED;
}
// get # of blocks
blocks = 0;
for (i=0 ; i<srccnt ; i++) {
uint32_t type,size;
uint64_t pchid;
uint32_t pver;
uint16_t pblocks;
uint8_t pstatus;
uint32_t ip;
rptr = r.repsources[i].hdrbuff;
type = get32bit(&rptr);
size = get32bit(&rptr);
rptr = r.repsources[i].packet;
ip = r.repsources[i].ip;
if (rptr==NULL || type!=CSTOAN_CHUNK_BLOCKS || size!=15) {
syslog(LOG_WARNING,"replicator,get # of blocks: got wrong answer (type:0x%08"PRIX32"/size:0x%08"PRIX32") from (%u.%u.%u.%u:%04"PRIX16")",type,size,(ip>>24)&0xFF,(ip>>16)&0xFF,(ip>>8)&0xFF,ip&0xFF,r.repsources[i].port);
rep_cleanup(&r);
return ERROR_DISCONNECTED;
}
pchid = get64bit(&rptr);
pver = get32bit(&rptr);
pblocks = get16bit(&rptr);
pstatus = get8bit(&rptr);
if (pchid!=r.repsources[i].chunkid) {
syslog(LOG_WARNING,"replicator,get # of blocks: got wrong answer (chunk_status:chunkid:%"PRIX64"/%"PRIX64") from (%u.%u.%u.%u:%04"PRIX16")",pchid,r.repsources[i].chunkid,(ip>>24)&0xFF,(ip>>16)&0xFF,(ip>>8)&0xFF,ip&0xFF,r.repsources[i].port);
rep_cleanup(&r);
return ERROR_WRONGCHUNKID;
}
