int gtmrecv_poll_actions1(int *pending_data_len, int *buff_unprocessed, unsigned char *buffp) { static int report_cnt = 1; static int next_report_at = 1; static boolean_t send_xoff = FALSE; static boolean_t xoff_sent = FALSE; static seq_num send_seqno; static boolean_t log_draining_msg = FALSE; static boolean_t send_badtrans = FALSE; static boolean_t send_cmp2uncmp = FALSE; static boolean_t upd_shut_too_early_logged = FALSE; static time_t last_reap_time = 0; repl_msg_t xoff_msg; repl_badtrans_msg_t bad_trans_msg; boolean_t alert = FALSE, info = FALSE; int return_status; gd_region *region_top; unsigned char *msg_ptr; /* needed for REPL_{SEND,RECV}_LOOP */ int tosend_len, sent_len, sent_this_iter; /* needed for REPL_SEND_LOOP */ int torecv_len, recvd_len, recvd_this_iter; /* needed for REPL_RECV_LOOP */ int status, poll_dir; /* needed for REPL_{SEND,RECV}_LOOP */ int temp_len, pending_msg_size; int upd_start_status, upd_start_attempts; int buffered_data_len; int upd_exit_status; seq_num temp_send_seqno; boolean_t bad_trans_detected = FALSE, onln_rlbk_flg_set = FALSE; recvpool_ctl_ptr_t recvpool_ctl; upd_proc_local_ptr_t upd_proc_local; gtmrecv_local_ptr_t gtmrecv_local; upd_helper_ctl_ptr_t upd_helper_ctl; pid_t waitpid_res; int4 msg_type, msg_len; DCL_THREADGBL_ACCESS; SETUP_THREADGBL_ACCESS; recvpool_ctl = recvpool.recvpool_ctl; upd_proc_local = recvpool.upd_proc_local; gtmrecv_local = recvpool.gtmrecv_local; upd_helper_ctl = recvpool.upd_helper_ctl; if (SHUTDOWN == gtmrecv_local->shutdown) { repl_log(gtmrecv_log_fp, TRUE, TRUE, "Shutdown signalled\n"); gtmrecv_end(); /* Won't return */ } # ifdef GTM_TLS /* If we sent a REPL_RENEG_ACK, then we cannot afford to send anymore asynchronous messages (like XOFF_ACK_ME) until we * receive a REPL_RENEG_COMPLETE from the source server. This ensures that while the source server attempts to do a SSL/TLS * renegotiation, it doesn't have any application data (like XOFF_ACK_ME) sitting in the pipe. */ if (REPLTLS_WAITING_FOR_RENEG_COMPLETE == repl_tls.renegotiate_state) return STOP_POLL; # endif /* Reset report_cnt and next_report_at to 1 when a new upd proc is forked */ if ((1 == report_cnt) || (report_cnt == next_report_at)) { /* A comment on the usage of NO_SHUTDOWN below for the alert variable. Since upd_proc_local->upd_proc_shutdown is * a shared memory field (and could be concurrently changed by either the receiver server or the update process), * we want to make sure it is the same value BEFORE and AFTER checking whether the update process is alive or not. * If it is not NO_SHUTDOWN (i.e. is SHUTDOWN or NORMAL_SHUTDOWN or ABNORMAL_SHUTDOWN) it has shut down due to * an external request so we do want to send out a false update-process-is-not-alive alert. */ if ((alert = ((NO_SHUTDOWN == upd_proc_local->upd_proc_shutdown) && (SRV_DEAD == is_updproc_alive()) && (NO_SHUTDOWN == upd_proc_local->upd_proc_shutdown))) || (info = (((NORMAL_SHUTDOWN == upd_proc_local->upd_proc_shutdown) || (ABNORMAL_SHUTDOWN == upd_proc_local->upd_proc_shutdown)) && (SRV_DEAD == is_updproc_alive())))) { if (alert) repl_log(gtmrecv_log_fp, TRUE, TRUE, "ALERT : Receiver Server detected that Update Process is not ALIVE\n"); else repl_log(gtmrecv_log_fp, TRUE, TRUE, "INFO : Update process not running due to user initiated shutdown\n"); if (1 == report_cnt) { send_xoff = TRUE; recvpool_ctl->old_jnl_seqno = recvpool_ctl->jnl_seqno; recvpool_ctl->jnl_seqno = 0; /* Even though we have identified that the update process is NOT alive, a waitpid on the update * process PID is necessary so that the system doesn't leave any zombie process lying around. * This is possible since any child process that dies without the parent doing a waitpid on it * will be defunct unless the parent dies at which point the "init" process takes the role of * the parent and invokes waitpid to remove the zombies. * NOTE: It is possible that the update process was killed before the receiver server got a * chance to record it's PID in the recvpool.upd_proc_local structure. In such a case, don't * invoke waitpid as that will block us (receiver server) if this instance of the receiver * server was started with helper processes. */ if (0 < upd_proc_local->upd_proc_pid) { WAITPID(upd_proc_local->upd_proc_pid, &upd_exit_status, 0, waitpid_res); /* Since the update process as part of its shutdown does NOT reset the upd_proc_pid, reset * it here ONLY if the update process was NOT kill -9ed. This is needed because receiver * server as part of its shutdown relies on this field (upd_proc_pid) to determine if the * update process was cleanly shutdown or was kill -9ed. */ if (!alert) upd_proc_local->upd_proc_pid = 0; } upd_proc_local->bad_trans = FALSE; /* No point in doing bad transaction processing */ upd_proc_local->onln_rlbk_flg = FALSE; /* No point handling online rollback */ } gtmrecv_wait_for_jnl_seqno = TRUE; REPL_DPRINT1( "gtmrecv_poll_actions : Setting gtmrecv_wait_for_jnl_seqno to TRUE because of upd crash/shutdown\n"); next_report_at *= GTMRECV_NEXT_REPORT_FACTOR; report_cnt++; } } else report_cnt++; /* Check if REPL_CMP2UNCMP or REPL_BADTRANS message needs to be sent */ if (upd_proc_local->onln_rlbk_flg) { /* Update process detected an online rollback and is requesting us to restart the connection. But before that, send * REPL_XOFF source side and drain the replication pipe */ onln_rlbk_flg_set = TRUE; send_xoff = TRUE; } else if (!send_cmp2uncmp && gtmrecv_send_cmp2uncmp) { send_xoff = TRUE; send_seqno = recvpool_ctl->jnl_seqno; send_cmp2uncmp = TRUE; } else if (!send_badtrans && upd_proc_local->bad_trans) { send_xoff = TRUE; send_seqno = upd_proc_local->read_jnl_seqno; send_badtrans = TRUE; bad_trans_detected = TRUE; } else if (!upd_proc_local->bad_trans && send_badtrans && 1 != report_cnt) { send_badtrans = FALSE; bad_trans_detected = FALSE; } if (send_xoff && !xoff_sent) { /* Send XOFF_ACK_ME if the receiver has a connection to the source. Do not attempt to send it if we dont even * know the endianness of the remote side. In that case, we are guaranteed no initial handshake occurred and * so no point sending the XOFF too. This saves us lots of trouble in case of cross-endian replication connections. */ assert((FD_INVALID != gtmrecv_sock_fd) || repl_connection_reset); if ((FD_INVALID != gtmrecv_sock_fd) && remote_side->endianness_known) { send_seqno = upd_proc_local->read_jnl_seqno; if (!remote_side->cross_endian) { xoff_msg.type = REPL_XOFF_ACK_ME; xoff_msg.len = MIN_REPL_MSGLEN; memcpy((uchar_ptr_t)&xoff_msg.msg[0], (uchar_ptr_t)&send_seqno, SIZEOF(seq_num)); } else { xoff_msg.type = GTM_BYTESWAP_32(REPL_XOFF_ACK_ME); xoff_msg.len = GTM_BYTESWAP_32(MIN_REPL_MSGLEN); temp_send_seqno = GTM_BYTESWAP_64(send_seqno); memcpy((uchar_ptr_t)&xoff_msg.msg[0], (uchar_ptr_t)&temp_send_seqno, SIZEOF(seq_num)); } REPL_SEND_LOOP(gtmrecv_sock_fd, &xoff_msg, MIN_REPL_MSGLEN, REPL_POLL_NOWAIT) ; /* Empty Body */ if (SS_NORMAL != status) { if (REPL_CONN_RESET(status) && EREPL_SEND == repl_errno) { repl_log(gtmrecv_log_fp, TRUE, TRUE, "Connection reset while sending XOFF_ACK_ME. " "Status = %d ; %s\n", status, STRERROR(status)); repl_close(>mrecv_sock_fd); repl_connection_reset = TRUE; xoff_sent = FALSE; send_badtrans = FALSE; } else if (EREPL_SEND == repl_errno) rts_error_csa(CSA_ARG(NULL) VARLSTCNT(7) ERR_REPLCOMM, 0, ERR_TEXT, 2, LEN_AND_LIT("Error sending XOFF msg due to BAD_TRANS or UPD crash/shutdown. " "Error in send"), status); else { assert(EREPL_SELECT == repl_errno); rts_error_csa(CSA_ARG(NULL) VARLSTCNT(7) ERR_REPLCOMM, 0, ERR_TEXT, 2, LEN_AND_LIT("Error sending XOFF msg due to BAD_TRANS or UPD crash/shutdown. " "Error in select"), status); } } else { xoff_sent = TRUE; log_draining_msg = TRUE; } repl_log(gtmrecv_log_fp, TRUE, TRUE, "REPL_XOFF_ACK_ME sent due to upd shutdown/crash or bad trans " "or ONLINE_ROLLBACK\n"); send_xoff = FALSE; } else { /* Connection has been lost OR initial handshake needs to happen again, so no point sending XOFF/BADTRANS */ send_xoff = FALSE; send_badtrans = FALSE; } } /* Drain pipe */ if (xoff_sent) { if (log_draining_msg) { /* avoid multiple logs per instance */ repl_log(gtmrecv_log_fp, TRUE, TRUE, "REPL INFO - Draining replication pipe due to %s\n", send_cmp2uncmp ? "CMP2UNCMP" : (send_badtrans ? "BAD_TRANS" : (onln_rlbk_flg_set ? "ONLINE_ROLLBACK" : "UPD shutdown/crash"))); log_draining_msg = FALSE; } if (0 != *buff_unprocessed) { /* Throw away the current contents of the buffer */ buffered_data_len = ((*pending_data_len <= *buff_unprocessed) ? *pending_data_len : *buff_unprocessed); *buff_unprocessed -= buffered_data_len; buffp += buffered_data_len; *pending_data_len -= buffered_data_len; REPL_DPRINT2("gtmrecv_poll_actions : (1) Throwing away %d bytes from old buffer while draining\n", buffered_data_len); assert(remote_side->endianness_known); /* only then is remote_side->cross_endian reliable */ while (REPL_MSG_HDRLEN <= *buff_unprocessed) { assert(0 == (((unsigned long)buffp) % REPL_MSG_ALIGN)); msg_len = ((repl_msg_ptr_t)buffp)->len; msg_type = ((repl_msg_ptr_t)buffp)->type; if (remote_side->cross_endian) { msg_len = GTM_BYTESWAP_32(msg_len); msg_type = GTM_BYTESWAP_32(msg_type); } msg_type = (msg_type & REPL_TR_CMP_MSG_TYPE_MASK); assert((REPL_TR_CMP_JNL_RECS == msg_type) || (0 == (msg_len % REPL_MSG_ALIGN))); *pending_data_len = ROUND_UP2(msg_len, REPL_MSG_ALIGN); buffered_data_len = ((*pending_data_len <= *buff_unprocessed) ? *pending_data_len : *buff_unprocessed); *buff_unprocessed -= buffered_data_len; buffp += buffered_data_len; *pending_data_len -= buffered_data_len; REPL_DPRINT3("gtmrecv_poll_actions : (1) Throwing away message of " "type %d and length %d from old buffer while draining\n", msg_type, buffered_data_len); } if (0 < *buff_unprocessed) { memmove((unsigned char *)gtmrecv_msgp, buffp, *buff_unprocessed); REPL_DPRINT2("gtmrecv_poll_actions : Incomplete header of length %d while draining\n", *buff_unprocessed); } } status = SS_NORMAL; if (0 != *buff_unprocessed || 0 == *pending_data_len) { /* Receive the header of a message */ assert(REPL_MSG_HDRLEN > *buff_unprocessed); /* so we dont pass negative length in REPL_RECV_LOOP */ REPL_RECV_LOOP(gtmrecv_sock_fd, ((unsigned char *)gtmrecv_msgp) + *buff_unprocessed, (REPL_MSG_HDRLEN - *buff_unprocessed), REPL_POLL_WAIT) ; /* Empty Body */ if (SS_NORMAL == status) { assert(remote_side->endianness_known); /* only then is remote_side->cross_endian reliable */ if (!remote_side->cross_endian) { msg_len = gtmrecv_msgp->len; msg_type = gtmrecv_msgp->type; } else { msg_len = GTM_BYTESWAP_32(gtmrecv_msgp->len); msg_type = GTM_BYTESWAP_32(gtmrecv_msgp->type); } msg_type = (msg_type & REPL_TR_CMP_MSG_TYPE_MASK); assert((REPL_TR_CMP_JNL_RECS == msg_type) || (0 == (msg_len % REPL_MSG_ALIGN))); msg_len = ROUND_UP2(msg_len, REPL_MSG_ALIGN); REPL_DPRINT3("gtmrecv_poll_actions : Received message of type %d and length %d while draining\n", msg_type, msg_len); } } if ((SS_NORMAL == status) && (0 != *buff_unprocessed || 0 == *pending_data_len) && (REPL_XOFF_ACK == msg_type)) { /* Receive the rest of the XOFF_ACK msg and signal the drain as complete */ REPL_RECV_LOOP(gtmrecv_sock_fd, gtmrecv_msgp, (MIN_REPL_MSGLEN - REPL_MSG_HDRLEN), REPL_POLL_WAIT) ; /* Empty Body */ if (SS_NORMAL == status) { repl_log(gtmrecv_log_fp, TRUE, TRUE, "REPL INFO - XOFF_ACK received. Drained replication pipe completely\n"); upd_shut_too_early_logged = FALSE; xoff_sent = FALSE; return_status = STOP_POLL; } } else if (SS_NORMAL == status) { /* Drain the rest of the message */ if (0 < *pending_data_len) { pending_msg_size = *pending_data_len; REPL_DPRINT2("gtmrecv_poll_actions : (2) Throwing away %d bytes from pipe\n", pending_msg_size); } else { pending_msg_size = msg_len - REPL_MSG_HDRLEN; REPL_DPRINT3("gtmrecv_poll_actions : (2) Throwing away message of " "type %d and length %d from pipe\n", msg_type, msg_len); } for ( ; SS_NORMAL == status && 0 < pending_msg_size; pending_msg_size -= gtmrecv_max_repl_msglen) { temp_len = (pending_msg_size < gtmrecv_max_repl_msglen)? pending_msg_size : gtmrecv_max_repl_msglen; REPL_RECV_LOOP(gtmrecv_sock_fd, gtmrecv_msgp, temp_len, REPL_POLL_WAIT) ; /* Empty Body */ } *buff_unprocessed = 0; *pending_data_len = 0; if (SS_NORMAL == status && info && !upd_shut_too_early_logged) { repl_log(gtmrecv_log_fp, TRUE, TRUE, "ALERT : User initiated shutdown of Update Process done " "when there was data in the replication pipe\n"); upd_shut_too_early_logged = TRUE; } return_status = CONTINUE_POLL; } if (SS_NORMAL != status) { if (EREPL_RECV == repl_errno) { if (REPL_CONN_RESET(status)) { repl_log(gtmrecv_log_fp, TRUE, TRUE, "Connection reset while receiving XOFF_ACK. " "Status = %d ; %s\n", status, STRERROR(status)); repl_close(>mrecv_sock_fd); repl_connection_reset = TRUE; xoff_sent = FALSE; send_badtrans = FALSE; return_status = STOP_POLL; } else rts_error_csa(CSA_ARG(NULL) VARLSTCNT(7) ERR_REPLCOMM, 0, ERR_TEXT, 2, LEN_AND_LIT("Error while draining replication pipe. Error in recv"), status); } else { assert(EREPL_SELECT == repl_errno); rts_error_csa(CSA_ARG(NULL) VARLSTCNT(7) ERR_REPLCOMM, 0, ERR_TEXT, 2, LEN_AND_LIT("Error while draining replication pipe. Error in select"), status); } } } else return_status = STOP_POLL; /* Like was done before for the XOFF_ACK_ME message, send a BADTRANS/CMP2UNCMP message only if we know * the endianness of the other side. If not, no point in sending one anyways and saves us trouble in * case of cross-endian replication connections. */ if ((STOP_POLL == return_status) && (send_badtrans || send_cmp2uncmp) && (FD_INVALID != gtmrecv_sock_fd) && remote_side->endianness_known) { /* Send REPL_BADTRANS or REPL_CMP2UNCMP message */ if (!remote_side->cross_endian) { bad_trans_msg.type = send_cmp2uncmp ? REPL_CMP2UNCMP : REPL_BADTRANS; bad_trans_msg.len = MIN_REPL_MSGLEN; bad_trans_msg.start_seqno = send_seqno; } else { bad_trans_msg.type = send_cmp2uncmp ? GTM_BYTESWAP_32(REPL_CMP2UNCMP) : GTM_BYTESWAP_32(REPL_BADTRANS); bad_trans_msg.len = GTM_BYTESWAP_32(MIN_REPL_MSGLEN); bad_trans_msg.start_seqno = GTM_BYTESWAP_64(send_seqno); } REPL_SEND_LOOP(gtmrecv_sock_fd, &bad_trans_msg, bad_trans_msg.len, REPL_POLL_NOWAIT) ; /* Empty Body */ if (SS_NORMAL == status) { if (send_cmp2uncmp) repl_log(gtmrecv_log_fp, TRUE, TRUE, "REPL_CMP2UNCMP message sent with seqno %llu\n", send_seqno); else repl_log(gtmrecv_log_fp, TRUE, TRUE, "REPL_BADTRANS message sent with seqno %llu\n", send_seqno); } else { if (REPL_CONN_RESET(status) && EREPL_SEND == repl_errno) { if (send_cmp2uncmp) { repl_log(gtmrecv_log_fp, TRUE, TRUE, "Connection reset while sending REPL_CMP2UNCMP. " "Status = %d ; %s\n", status, STRERROR(status)); } else { repl_log(gtmrecv_log_fp, TRUE, TRUE, "Connection reset while sending REPL_BADTRANS. " "Status = %d ; %s\n", status, STRERROR(status)); } repl_close(>mrecv_sock_fd); repl_connection_reset = TRUE; return_status = STOP_POLL; } else if (EREPL_SEND == repl_errno) rts_error_csa(CSA_ARG(NULL) VARLSTCNT(7) ERR_REPLCOMM, 0, ERR_TEXT, 2, LEN_AND_LIT("Error sending REPL_BADTRANS/REPL_CMP2UNCMP. Error in send"), status); else { assert(EREPL_SELECT == repl_errno); rts_error_csa(CSA_ARG(NULL) VARLSTCNT(7) ERR_REPLCOMM, 0, ERR_TEXT, 2, LEN_AND_LIT("Error sending REPL_BADTRANS/REPL_CMP2UNCMP. Error in select"), status); } } send_badtrans = FALSE; if (send_cmp2uncmp) { REPL_DPRINT1("gtmrecv_poll_actions : Setting gtmrecv_wait_for_jnl_seqno to TRUE because this receiver" "server requested a fall-back from compressed to uncompressed operation\n"); gtmrecv_wait_for_jnl_seqno = TRUE;/* set this to TRUE to break out and go back to a fresh "do_main_loop" */ gtmrecv_bad_trans_sent = TRUE; gtmrecv_send_cmp2uncmp = FALSE; send_cmp2uncmp = FALSE; } } if ((upd_proc_local->bad_trans && bad_trans_detected) || onln_rlbk_flg_set || (UPDPROC_START == upd_proc_local->start_upd) && (1 != report_cnt)) { if (UPDPROC_START == upd_proc_local->start_upd) { assert(is_updproc_alive() != SRV_ALIVE); upd_proc_local->upd_proc_shutdown = NO_SHUTDOWN; } recvpool_ctl->wrapped = FALSE; recvpool_ctl->write_wrap = recvpool_ctl->recvpool_size; recvpool_ctl->write = 0; /* Reset last_rcvd_histinfo, last_valid_histinfo etc. as they reflect context from unprocessed data * in the receive pool and those are no longer valid because we have drained the receive pool. */ GTMRECV_CLEAR_CACHED_HISTINFO(recvpool.recvpool_ctl, jnlpool, jnlpool_ctl, INSERT_STRM_HISTINFO_FALSE); if (UPDPROC_START == upd_proc_local->start_upd) { /* Attempt starting the update process */ for (upd_start_attempts = 0; UPDPROC_START_ERR == (upd_start_status = gtmrecv_upd_proc_init(FALSE)) && GTMRECV_MAX_UPDSTART_ATTEMPTS > upd_start_attempts; upd_start_attempts++) { if (EREPL_UPDSTART_SEMCTL == repl_errno || EREPL_UPDSTART_BADPATH == repl_errno) { gtmrecv_autoshutdown(); } else if (EREPL_UPDSTART_FORK == repl_errno) { /* Couldn't start up update now, can try later */ LONG_SLEEP(GTMRECV_WAIT_FOR_PROC_SLOTS); continue; } else if (EREPL_UPDSTART_EXEC == repl_errno) { /* In forked child, could not exec, should exit */ gtmrecv_exit(ABNORMAL_SHUTDOWN); } } if (UPDPROC_STARTED == (upd_proc_local->start_upd = upd_start_status)) { REPL_DPRINT1("gtmrecv_poll_actions : Setting gtmrecv_wait_for_jnl_seqno to TRUE because of " "upd restart\n"); gtmrecv_wait_for_jnl_seqno = TRUE; report_cnt = next_report_at = 1; if (send_xoff && (FD_INVALID == gtmrecv_sock_fd)) { /* Update start command was issued before connection was established, * no point in sending XOFF. */ send_xoff = FALSE; } } else { repl_log(gtmrecv_log_fp, TRUE, TRUE, "%d failed attempts to fork update process. Try later\n", upd_start_attempts); } } else { gtmrecv_wait_for_jnl_seqno = TRUE;/* set this to TRUE to break out and go back to a fresh "do_main_loop" */ if (onln_rlbk_flg_set) { assert(NULL != jnlpool_ctl); repl_log(gtmrecv_log_fp, TRUE, TRUE, "Closing connection due to ONLINE ROLLBACK\n"); repl_log(gtmrecv_log_fp, TRUE, TRUE, "REPL INFO - Current Jnlpool Seqno : %llu\n", jnlpool_ctl->jnl_seqno); repl_log(gtmrecv_log_fp, TRUE, TRUE, "REPL INFO - Current Receive Pool Seqno : %llu\n", recvpool_ctl->jnl_seqno); repl_close(>mrecv_sock_fd); repl_connection_reset = TRUE; xoff_sent = FALSE; send_badtrans = FALSE; upd_proc_local->onln_rlbk_flg = FALSE; /* Before restarting afresh, sync the online rollback cycles. This way any future grab_lock that * we do after restarting should not realize an unhandled online rollback. For receiver, it is * just syncing the journal pool cycles as the databases are not opened. But, to be safe, grab * the lock and sync the cycles. */ grab_lock(jnlpool.jnlpool_dummy_reg, TRUE, GRAB_LOCK_ONLY); SYNC_ONLN_RLBK_CYCLES; rel_lock(jnlpool.jnlpool_dummy_reg); return_status = STOP_POLL; recvpool_ctl->jnl_seqno = 0; } else { REPL_DPRINT1("gtmrecv_poll_actions : Setting gtmrecv_wait_for_jnl_seqno to TRUE because bad trans" "sent\n"); gtmrecv_bad_trans_sent = TRUE; upd_proc_local->bad_trans = FALSE; recvpool_ctl->jnl_seqno = upd_proc_local->read_jnl_seqno; } } } if ((0 == *pending_data_len) && (0 != gtmrecv_local->changelog)) { if (gtmrecv_local->changelog & REPLIC_CHANGE_LOGINTERVAL) { repl_log(gtmrecv_log_fp, TRUE, TRUE, "Changing log interval from %u to %u\n", log_interval, gtmrecv_local->log_interval); log_interval = gtmrecv_local->log_interval; gtmrecv_reinit_logseqno(); /* will force a LOG on the first recv following the interval change */ } if (gtmrecv_local->changelog & REPLIC_CHANGE_LOGFILE) { repl_log(gtmrecv_log_fp, TRUE, TRUE, "Changing log file to %s\n", gtmrecv_local->log_file); repl_log_init(REPL_GENERAL_LOG, >mrecv_log_fd, gtmrecv_local->log_file); repl_log_fd2fp(>mrecv_log_fp, gtmrecv_log_fd); repl_log(gtmrecv_log_fp, TRUE, TRUE, "Change log to %s successful\n",gtmrecv_local->log_file); } /* NOTE: update process and receiver each ignore any setting specific to the other (REPLIC_CHANGE_UPD_LOGINTERVAL, * REPLIC_CHANGE_LOGINTERVAL) */ if (REPLIC_CHANGE_LOGINTERVAL == gtmrecv_local->changelog) upd_proc_local->changelog = 0; else upd_proc_local->changelog = gtmrecv_local->changelog; /* Pass changelog request to the update process */ gtmrecv_local->changelog = 0; } if (0 == *pending_data_len && !gtmrecv_logstats && gtmrecv_local->statslog) { gtmrecv_logstats = TRUE; repl_log(gtmrecv_log_fp, TRUE, TRUE, "Begin statistics logging\n"); } else if (0 == *pending_data_len && gtmrecv_logstats && !gtmrecv_local->statslog) { gtmrecv_logstats = FALSE; /* Force all data out to the file before closing the file */ repl_log(gtmrecv_log_fp, TRUE, TRUE, "End statistics logging\n"); } if (0 == *pending_data_len) { if (upd_helper_ctl->start_helpers) { gtmrecv_helpers_init(upd_helper_ctl->start_n_readers, upd_helper_ctl->start_n_writers); upd_helper_ctl->start_helpers = FALSE; } if (HELPER_REAP_NONE != (status = upd_helper_ctl->reap_helpers) || (double)GTMRECV_REAP_HELPERS_INTERVAL <= difftime(gtmrecv_now, last_reap_time)) { gtmrecv_reap_helpers(HELPER_REAP_WAIT == status); last_reap_time = gtmrecv_now; } } return (return_status); }
int gtmsource() { int status, log_init_status, waitpid_res, save_errno; char print_msg[1024], tmpmsg[1024]; gd_region *reg, *region_top; sgmnt_addrs *csa, *repl_csa; boolean_t all_files_open, isalive; pid_t pid, ppid, procgp; seq_num read_jnl_seqno, jnl_seqno; unix_db_info *udi; gtmsource_local_ptr_t gtmsource_local; boolean_t this_side_std_null_coll; int null_fd, rc; memset((uchar_ptr_t)&jnlpool, 0, SIZEOF(jnlpool_addrs)); call_on_signal = gtmsource_sigstop; ESTABLISH_RET(gtmsource_ch, SS_NORMAL); if (-1 == gtmsource_get_opt()) rts_error_csa(CSA_ARG(NULL) VARLSTCNT(1) ERR_MUPCLIERR); if (gtmsource_options.shut_down) { /* Wait till shutdown time nears even before going to "jnlpool_init". This is because the latter will return * with the ftok semaphore and access semaphore held and we do not want to be holding those locks (while * waiting for the user specified timeout to expire) as that will affect new GTM processes and/or other * MUPIP REPLIC commands that need these locks for their function. */ if (0 < gtmsource_options.shutdown_time) { repl_log(stdout, TRUE, TRUE, "Waiting for %d seconds before signalling shutdown\n", gtmsource_options.shutdown_time); LONG_SLEEP(gtmsource_options.shutdown_time); } else repl_log(stdout, TRUE, TRUE, "Signalling shutdown immediate\n"); } else if (gtmsource_options.start) { repl_log(stdout, TRUE, TRUE, "Initiating START of source server for secondary instance [%s]\n", gtmsource_options.secondary_instname); } if (gtmsource_options.activate && (ROOTPRIMARY_SPECIFIED == gtmsource_options.rootprimary)) { /* MUPIP REPLIC -SOURCE -ACTIVATE -UPDOK has been specified. We need to open the gld and db regions now * in case this is a secondary -> primary transition. This is so we can later switch journal files in all * journaled regions when the transition actually happens inside "gtmsource_rootprimary_init". But since * we have not yet done a "jnlpool_init", we dont know if updates are disabled in it or not. Although we * need to do the gld/db open only if updates are currently disabled in the jnlpool, we do this always * because once we do a jnlpool_init, we will come back with the ftok on the jnlpool held and that has * issues with later db open since we will try to hold the db ftok as part of db open and the ftok logic * currently has assumptions that a process holds only one ftok at any point in time. */ assert(NULL == gd_header); gvinit(); all_files_open = region_init(FALSE); if (!all_files_open) { gtm_putmsg_csa(CSA_ARG(NULL) VARLSTCNT(1) ERR_NOTALLDBOPN); gtmsource_exit(ABNORMAL_SHUTDOWN); } } jnlpool_init(GTMSOURCE, gtmsource_options.start, &is_jnlpool_creator); /* is_jnlpool_creator == TRUE ==> this process created the journal pool * is_jnlpool_creator == FALSE ==> journal pool already existed and this process simply attached to it. */ if (gtmsource_options.shut_down) gtmsource_exit(gtmsource_shutdown(FALSE, NORMAL_SHUTDOWN) - NORMAL_SHUTDOWN); else if (gtmsource_options.activate) gtmsource_exit(gtmsource_mode_change(GTMSOURCE_MODE_ACTIVE_REQUESTED) - NORMAL_SHUTDOWN); else if (gtmsource_options.deactivate) gtmsource_exit(gtmsource_mode_change(GTMSOURCE_MODE_PASSIVE_REQUESTED) - NORMAL_SHUTDOWN); else if (gtmsource_options.checkhealth) gtmsource_exit(gtmsource_checkhealth() - NORMAL_SHUTDOWN); else if (gtmsource_options.changelog) gtmsource_exit(gtmsource_changelog() - NORMAL_SHUTDOWN); else if (gtmsource_options.showbacklog) gtmsource_exit(gtmsource_showbacklog() - NORMAL_SHUTDOWN); else if (gtmsource_options.stopsourcefilter) gtmsource_exit(gtmsource_stopfilter() - NORMAL_SHUTDOWN); else if (gtmsource_options.jnlpool) gtmsource_exit(gtmsource_jnlpool() - NORMAL_SHUTDOWN); else if (gtmsource_options.losttncomplete) gtmsource_exit(gtmsource_losttncomplete() - NORMAL_SHUTDOWN); else if (gtmsource_options.needrestart) gtmsource_exit(gtmsource_needrestart() - NORMAL_SHUTDOWN); else if (gtmsource_options.showfreeze) gtmsource_exit(gtmsource_showfreeze() - NORMAL_SHUTDOWN); else if (gtmsource_options.setfreeze) gtmsource_exit(gtmsource_setfreeze() - NORMAL_SHUTDOWN); else if (!gtmsource_options.start) { assert(CLI_PRESENT == cli_present("STATSLOG")); gtmsource_exit(gtmsource_statslog() - NORMAL_SHUTDOWN); } assert(gtmsource_options.start); # ifndef REPL_DEBUG_NOBACKGROUND /* Set "child_server_running" to FALSE before forking off child. Wait for it to be set to TRUE by the child. */ gtmsource_local = jnlpool.gtmsource_local; gtmsource_local->child_server_running = FALSE; FORK(pid); if (0 > pid) { save_errno = errno; rts_error_csa(CSA_ARG(NULL) VARLSTCNT(7) ERR_JNLPOOLSETUP, 0, ERR_TEXT, 2, RTS_ERROR_LITERAL("Could not fork source server"), save_errno); } else if (0 < pid) { /* Parent. Wait until child sets "child_server_running" to FALSE. That is an indication that the child * source server has completed its initialization phase and is all set so the parent command can return. */ while (isalive = is_proc_alive(pid, 0)) /* note : intended assignment */ { if (gtmsource_local->child_server_running) break; /* To take care of reassignment of PIDs, the while condition should be && with the condition * (PPID of pid == process_id) */ SHORT_SLEEP(GTMSOURCE_WAIT_FOR_SRV_START); WAITPID(pid, &status, WNOHANG, waitpid_res); /* Release defunct child if dead */ } if (isalive) { /* Child process is alive and started with no issues */ if (0 != (save_errno = rel_sem(SOURCE, JNL_POOL_ACCESS_SEM))) rts_error_csa(CSA_ARG(NULL) VARLSTCNT(7) ERR_JNLPOOLSETUP, 0, ERR_TEXT, 2, RTS_ERROR_LITERAL("Error in rel_sem"), save_errno); ftok_sem_release(jnlpool.jnlpool_dummy_reg, TRUE, TRUE); } else { /* Child source server process errored out at startup and is no longer alive. * If we were the one who created the journal pool, let us clean it up. */ repl_log(stdout, TRUE, TRUE, "Source server startup failed. See source server log file\n"); if (is_jnlpool_creator) status = gtmsource_shutdown(TRUE, NORMAL_SHUTDOWN); } /* If the parent is killed (or crashes) between the fork and exit, checkhealth may not detect that startup * is in progress - parent forks and dies, the system will release sem 0 and 1, checkhealth might test the * value of sem 1 before the child grabs sem 1. */ gtmsource_exit(isalive ? SRV_ALIVE : SRV_ERR); } /* Point stdin to /dev/null */ OPENFILE("/dev/null", O_RDONLY, null_fd); if (0 > null_fd) rts_error_csa(CSA_ARG(NULL) ERR_REPLERR, RTS_ERROR_LITERAL("Failed to open /dev/null for read"), errno, 0); FCNTL3(null_fd, F_DUPFD, 0, rc); if (0 > rc) rts_error_csa(CSA_ARG(NULL) ERR_REPLERR, RTS_ERROR_LITERAL("Failed to set stdin to /dev/null"), errno, 0); CLOSEFILE(null_fd, rc); if (0 > rc) rts_error_csa(CSA_ARG(NULL) ERR_REPLERR, RTS_ERROR_LITERAL("Failed to close /dev/null"), errno, 0); /* The parent process (source server startup command) will be holding the ftok semaphore and jnlpool access semaphore * at this point. The variables that indicate this would have been copied over to the child during the fork. This will * make the child think it is actually holding them as well when actually it is not. Reset those variables in the child * to ensure they do not misrepresent the holder of those semaphores. */ ftok_sem_reg = NULL; udi = FILE_INFO(jnlpool.jnlpool_dummy_reg); assert(udi->grabbed_ftok_sem); udi->grabbed_ftok_sem = FALSE; assert(holds_sem[SOURCE][JNL_POOL_ACCESS_SEM]); holds_sem[SOURCE][JNL_POOL_ACCESS_SEM] = FALSE; assert(!holds_sem[SOURCE][SRC_SERV_COUNT_SEM]); /* Start child source server initialization */ is_src_server = TRUE; OPERATOR_LOG_MSG; process_id = getpid(); /* Reinvoke secshr related initialization with the child's pid */ INVOKE_INIT_SECSHR_ADDRS; /* Initialize mutex socket, memory semaphore etc. before any "grab_lock" is done by this process on the journal pool. * Note that the initialization would already have been done by the parent receiver startup command but we need to * redo the initialization with the child process id. */ assert(mutex_per_process_init_pid && (mutex_per_process_init_pid != process_id)); mutex_per_process_init(); START_HEARTBEAT_IF_NEEDED; ppid = getppid(); log_init_status = repl_log_init(REPL_GENERAL_LOG, >msource_log_fd, gtmsource_options.log_file); assert(SS_NORMAL == log_init_status); repl_log_fd2fp(>msource_log_fp, gtmsource_log_fd); if (-1 == (procgp = setsid())) send_msg_csa(CSA_ARG(NULL) VARLSTCNT(7) ERR_JNLPOOLSETUP, 0, ERR_TEXT, 2, RTS_ERROR_LITERAL("Source server error in setsid"), errno); # endif /* REPL_DEBUG_NOBACKGROUND */ if (ZLIB_CMPLVL_NONE != gtm_zlib_cmp_level) gtm_zlib_init(); /* Open zlib shared library for compression/decompression */ REPL_DPRINT1("Setting up regions\n"); gvinit(); /* We use the same code dse uses to open all regions but we must make sure they are all open before proceeding. */ all_files_open = region_init(FALSE); if (!all_files_open) { gtm_putmsg_csa(CSA_ARG(NULL) VARLSTCNT(1) ERR_NOTALLDBOPN); gtmsource_exit(ABNORMAL_SHUTDOWN); } /* Determine primary side null subscripts collation order */ /* Also check whether all regions have same null collation order */ this_side_std_null_coll = -1; for (reg = gd_header->regions, region_top = gd_header->regions + gd_header->n_regions; reg < region_top; reg++) { csa = &FILE_INFO(reg)->s_addrs; if (this_side_std_null_coll != csa->hdr->std_null_coll) { if (-1 == this_side_std_null_coll) this_side_std_null_coll = csa->hdr->std_null_coll; else { gtm_putmsg_csa(CSA_ARG(NULL) VARLSTCNT(1) ERR_NULLCOLLDIFF); gtmsource_exit(ABNORMAL_SHUTDOWN); } } if (!REPL_ALLOWED(csa) && JNL_ALLOWED(csa)) { gtm_putmsg_csa(CSA_ARG(NULL) VARLSTCNT(4) ERR_REPLOFFJNLON, 2, DB_LEN_STR(reg)); gtmsource_exit(ABNORMAL_SHUTDOWN); } if (reg->read_only && REPL_ALLOWED(csa)) { gtm_putmsg_csa(CSA_ARG(NULL) VARLSTCNT(6) ERR_JNLPOOLSETUP, 0, ERR_TEXT, 2, RTS_ERROR_LITERAL("Source Server does not have write permissions to one or " "more database files that are replicated")); gtmsource_exit(ABNORMAL_SHUTDOWN); } } /* Initialize source server alive/dead state related fields in "gtmsource_local" before the ftok semaphore is released */ gtmsource_local->gtmsource_pid = process_id; gtmsource_local->gtmsource_state = GTMSOURCE_START; if (is_jnlpool_creator) { DEBUG_ONLY(jnlpool.jnlpool_ctl->jnlpool_creator_pid = process_id); gtmsource_seqno_init(this_side_std_null_coll); if (ROOTPRIMARY_SPECIFIED == gtmsource_options.rootprimary) { /* Created the journal pool as a root primary. Append a history record to the replication instance file. * Invoke the function "gtmsource_rootprimary_init" to do that. */ gtmsource_rootprimary_init(jnlpool.jnlpool_ctl->jnl_seqno); } } /* after this point we can no longer have the case where all the regions are unreplicated/non-journaled. */ # ifndef REPL_DEBUG_NOBACKGROUND /* It is necessary for every process that is using the ftok semaphore to increment the counter by 1. This is used * by the last process that shuts down to delete the ftok semaphore when it notices the counter to be 0. * Note that the parent source server startup command would have done an increment of the ftok counter semaphore * for the replication instance file. But the source server process (the child) that comes here would not have done * that. Do that while the parent is still holding on to the ftok semaphore waiting for our okay. */ if (!ftok_sem_incrcnt(jnlpool.jnlpool_dummy_reg)) rts_error_csa(CSA_ARG(NULL) VARLSTCNT(1) ERR_JNLPOOLSETUP); /* Increment the source server count semaphore */ status = incr_sem(SOURCE, SRC_SERV_COUNT_SEM); if (0 != status) { save_errno = errno; rts_error_csa(CSA_ARG(NULL) VARLSTCNT(7) ERR_JNLPOOLSETUP, 0, ERR_TEXT, 2, RTS_ERROR_LITERAL("Counter semaphore increment failure in child source server"), save_errno); } # else if (0 != (save_errno = rel_sem_immediate(SOURCE, JNL_POOL_ACCESS_SEM))) { rts_error_csa(CSA_ARG(NULL) VARLSTCNT(7) ERR_JNLPOOLSETUP, 0, ERR_TEXT, 2, RTS_ERROR_LITERAL("Error in rel_sem_immediate"), save_errno); } # endif /* REPL_DEBUG_NOBACKGROUND */ gtmsource_srv_count++; gtmsource_local->child_server_running = TRUE; /* At this point, the parent startup command will stop waiting for child */ gtm_event_log_init(); /* Log source server startup command line first */ SPRINTF(tmpmsg, "%s %s\n", cli_lex_in_ptr->argv[0], cli_lex_in_ptr->in_str); repl_log(gtmsource_log_fp, TRUE, TRUE, tmpmsg); SPRINTF(tmpmsg, "GTM Replication Source Server with Pid [%d] started for Secondary Instance [%s]", process_id, gtmsource_local->secondary_instname); sgtm_putmsg(print_msg, VARLSTCNT(4) ERR_REPLINFO, 2, LEN_AND_STR(tmpmsg)); repl_log(gtmsource_log_fp, TRUE, TRUE, print_msg); if (is_jnlpool_creator) { repl_log(gtmsource_log_fp, TRUE, TRUE, "Created jnlpool with shmid = [%d] and semid = [%d]\n", jnlpool.repl_inst_filehdr->jnlpool_shmid, jnlpool.repl_inst_filehdr->jnlpool_semid); } else repl_log(gtmsource_log_fp, TRUE, TRUE, "Attached to existing jnlpool with shmid = [%d] and semid = [%d]\n", jnlpool.repl_inst_filehdr->jnlpool_shmid, jnlpool.repl_inst_filehdr->jnlpool_semid); gtm_event_log(GTM_EVENT_LOG_ARGC, "MUPIP", "REPLINFO", print_msg); # ifdef GTM_TLS if (REPL_TLS_REQUESTED) { repl_do_tls_init(gtmsource_log_fp); assert(REPL_TLS_REQUESTED || PLAINTEXT_FALLBACK); } # endif if (jnlpool.jnlpool_ctl->freeze) { last_seen_freeze_flag = jnlpool.jnlpool_ctl->freeze; sgtm_putmsg(print_msg, VARLSTCNT(3) ERR_REPLINSTFROZEN, 1, jnlpool.repl_inst_filehdr->inst_info.this_instname); repl_log(gtmsource_log_fp, TRUE, FALSE, print_msg); sgtm_putmsg(print_msg, VARLSTCNT(3) ERR_REPLINSTFREEZECOMMENT, 1, jnlpool.jnlpool_ctl->freeze_comment); repl_log(gtmsource_log_fp, TRUE, TRUE, print_msg); } gtmsource_local->jnlfileonly = gtmsource_options.jnlfileonly; do { /* If mode is passive, go to sleep. Wakeup every now and then and check to see if I have to become active. */ gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_START; if ((gtmsource_local->mode == GTMSOURCE_MODE_PASSIVE) && (gtmsource_local->shutdown == NO_SHUTDOWN)) { gtmsource_poll_actions(FALSE); SHORT_SLEEP(GTMSOURCE_WAIT_FOR_MODE_CHANGE); continue; } if (GTMSOURCE_MODE_PASSIVE == gtmsource_local->mode) { /* Shutdown initiated */ assert(gtmsource_local->shutdown == SHUTDOWN); sgtm_putmsg(print_msg, VARLSTCNT(4) ERR_REPLINFO, 2, RTS_ERROR_LITERAL("GTM Replication Source Server Shutdown signalled")); repl_log(gtmsource_log_fp, TRUE, TRUE, print_msg); gtm_event_log(GTM_EVENT_LOG_ARGC, "MUPIP", "REPLINFO", print_msg); break; } gtmsource_poll_actions(FALSE); if (GTMSOURCE_CHANGING_MODE == gtmsource_state) continue; if (GTMSOURCE_MODE_ACTIVE_REQUESTED == gtmsource_local->mode) gtmsource_local->mode = GTMSOURCE_MODE_ACTIVE; SPRINTF(tmpmsg, "GTM Replication Source Server now in ACTIVE mode using port %d", gtmsource_local->secondary_port); sgtm_putmsg(print_msg, VARLSTCNT(4) ERR_REPLINFO, 2, LEN_AND_STR(tmpmsg)); repl_log(gtmsource_log_fp, TRUE, TRUE, print_msg); gtm_event_log(GTM_EVENT_LOG_ARGC, "MUPIP", "REPLINFO", print_msg); DEBUG_ONLY(repl_csa = &FILE_INFO(jnlpool.jnlpool_dummy_reg)->s_addrs;) assert(!repl_csa->hold_onto_crit); /* so it is ok to invoke "grab_lock" and "rel_lock" unconditionally */ grab_lock(jnlpool.jnlpool_dummy_reg, TRUE, HANDLE_CONCUR_ONLINE_ROLLBACK); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) { repl_log(gtmsource_log_fp, TRUE, TRUE, "Starting afresh due to ONLINE ROLLBACK\n"); repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL INFO - Current Jnlpool Seqno : %llu\n", jnlpool.jnlpool_ctl->jnl_seqno); continue; } QWASSIGN(gtmsource_local->read_addr, jnlpool.jnlpool_ctl->write_addr); gtmsource_local->read = jnlpool.jnlpool_ctl->write; gtmsource_local->read_state = gtmsource_local->jnlfileonly ? READ_FILE : READ_POOL; read_jnl_seqno = gtmsource_local->read_jnl_seqno; assert(read_jnl_seqno <= jnlpool.jnlpool_ctl->jnl_seqno); if (read_jnl_seqno < jnlpool.jnlpool_ctl->jnl_seqno) { gtmsource_local->read_state = READ_FILE; QWASSIGN(gtmsource_save_read_jnl_seqno, jnlpool.jnlpool_ctl->jnl_seqno); gtmsource_pool2file_transition = TRUE; /* so that we read the latest gener jnl files */ } rel_lock(jnlpool.jnlpool_dummy_reg); if (SS_NORMAL != (status = gtmsource_alloc_tcombuff())) rts_error_csa(CSA_ARG(NULL) VARLSTCNT(7) ERR_REPLCOMM, 0, ERR_TEXT, 2, RTS_ERROR_LITERAL("Error allocating initial tcom buffer space. Malloc error"), status); gtmsource_filter = NO_FILTER; if ('\0' != gtmsource_local->filter_cmd[0]) { if (SS_NORMAL == (status = repl_filter_init(gtmsource_local->filter_cmd))) gtmsource_filter |= EXTERNAL_FILTER; else gtmsource_exit(ABNORMAL_SHUTDOWN); } gtmsource_process(); /* gtmsource_process returns only when mode needs to be changed to PASSIVE */ assert(gtmsource_state == GTMSOURCE_CHANGING_MODE); gtmsource_ctl_close(); gtmsource_free_msgbuff(); gtmsource_free_tcombuff(); gtmsource_free_filter_buff(); gtmsource_stop_heartbeat(); if (FD_INVALID != gtmsource_sock_fd) repl_close(>msource_sock_fd); if (gtmsource_filter & EXTERNAL_FILTER) repl_stop_filter(); } while (TRUE);