void jnl_file_lost(jnl_private_control *jpc, uint4 jnl_stat)
{ /* Notify operator and terminate journaling */
unsigned int status;
sgmnt_addrs *csa;
seq_num reg_seqno, jnlseqno;
error_def(ERR_REPLJNLCLOSED);
error_def(ERR_JNLCLOSED);
switch(jpc->region->dyn.addr->acc_meth)
{
case dba_mm:
case dba_bg:
csa = &FILE_INFO(jpc->region)->s_addrs;
break;
default:
GTMASSERT;
}
#ifdef VMS
/* The following assert has been removed as it could be FALSE if the caller is "jnl_file_extend"
* assert(0 != memcmp(csa->nl->jnl_file.jnl_file_id.fid, zero_fid, sizeof(zero_fid)));
*/
#endif
assert(csa->now_crit);
if (0 != jnl_stat)
jnl_send_oper(jpc, jnl_stat);
csa->hdr->jnl_state = jnl_closed;
jpc->jnl_buff->cycle++; /* increment shared cycle so all future callers of jnl_ensure_open recognize journal switch */
assert(jpc->cycle < jpc->jnl_buff->cycle);
if (REPL_ENABLED(csa->hdr))
{
csa->hdr->repl_state = repl_was_open;
reg_seqno = csa->hdr->reg_seqno;
jnlseqno = (NULL != jnlpool.jnlpool_ctl) ? jnlpool.jnlpool_ctl->jnl_seqno : MAX_SEQNO;
send_msg(VARLSTCNT(8) ERR_REPLJNLCLOSED, 6, DB_LEN_STR(jpc->region), ®_seqno, ®_seqno, &jnlseqno, &jnlseqno);
} else
send_msg(VARLSTCNT(5) ERR_JNLCLOSED, 3, DB_LEN_STR(jpc->region), &csa->ti->curr_tn);
#ifdef VMS
assert(0 != csa->jnl->jnllsb->lockid);
status = gtm_enqw(EFN$C_ENF, LCK$K_EXMODE, csa->jnl->jnllsb, LCK$M_CONVERT | LCK$M_NODLCKBLK,
NULL, 0, NULL, 0, NULL, PSL$C_USER, 0);
if (SS$_NORMAL == status)
status = csa->jnl->jnllsb->cond;
jnl_file_close(jpc->region, FALSE, FALSE);
if (SS$_NORMAL == status)
status = gtm_deq(csa->jnl->jnllsb->lockid, NULL, PSL$C_USER, 0);
if (SS$_NORMAL != status)
GTMASSERT;
# else
jnl_file_close(jpc->region, FALSE, FALSE);
#endif
}
uint4 jnl_file_lost(jnl_private_control *jpc, uint4 jnl_stat)
{ /* Notify operator and terminate journaling */
unsigned int status;
sgmnt_addrs *csa;
seq_num reg_seqno, jnlseqno;
boolean_t was_lockid = FALSE, instfreeze_environ;
DCL_THREADGBL_ACCESS;
SETUP_THREADGBL_ACCESS;
switch(jpc->region->dyn.addr->acc_meth)
{
case dba_mm:
case dba_bg:
csa = &FILE_INFO(jpc->region)->s_addrs;
break;
default:
assertpro(FALSE && jpc->region->dyn.addr->acc_meth);
}
# ifdef VMS
/* The following assert has been removed as it could be FALSE if the caller is "jnl_file_extend"
* assert(0 != memcmp(csa->nl->jnl_file.jnl_file_id.fid, zero_fid, SIZEOF(zero_fid)));
*/
# endif
assert(csa->now_crit);
/* We issue an rts_error (instead of shutting off journaling) in the following cases : {BYPASSOK}
* 1) $gtm_error_on_jnl_file_lost is set to issue runtime error (if not already issued) in case of journaling issues.
* 2) The process has the given message set in $gtm_custom_errors (indicative of instance freeze on error setup)
* in which case the goal is to never shut-off journaling
*/
UNIX_ONLY(assert(jnlpool.jnlpool_ctl == jnlpool_ctl));
UNIX_ONLY(instfreeze_environ = INST_FREEZE_ON_MSG_ENABLED(csa, jnl_stat));
VMS_ONLY(instfreeze_environ = FALSE);
if ((JNL_FILE_LOST_ERRORS == TREF(error_on_jnl_file_lost)) || instfreeze_environ)
{
VMS_ONLY(assert(FALSE)); /* Not fully implemented / supported on VMS. */
if (!process_exiting || instfreeze_environ || !csa->jnl->error_reported)
{
csa->jnl->error_reported = TRUE;
in_wcs_recover = FALSE; /* in case we're called in wcs_recover() */
if (SS_NORMAL != jpc->status)
rts_error_csa(CSA_ARG(csa) VARLSTCNT(7) jnl_stat, 4, JNL_LEN_STR(csa->hdr),
DB_LEN_STR(gv_cur_region), jpc->status);
else
rts_error_csa(CSA_ARG(csa) VARLSTCNT(6) jnl_stat, 4, JNL_LEN_STR(csa->hdr),
DB_LEN_STR(gv_cur_region));
}
return jnl_stat;
}
if (0 != jnl_stat)
jnl_send_oper(jpc, jnl_stat);
csa->hdr->jnl_state = jnl_closed;
jpc->jnl_buff->cycle++; /* increment shared cycle so all future callers of jnl_ensure_open recognize journal switch */
assert(jpc->cycle < jpc->jnl_buff->cycle);
if (REPL_ENABLED(csa->hdr))
{
csa->hdr->repl_state = repl_was_open;
reg_seqno = csa->hdr->reg_seqno;
jnlseqno = (NULL != jnlpool.jnlpool_ctl) ? jnlpool.jnlpool_ctl->jnl_seqno : MAX_SEQNO;
send_msg_csa(CSA_ARG(csa) VARLSTCNT(8) ERR_REPLJNLCLOSED, 6, DB_LEN_STR(jpc->region), ®_seqno, ®_seqno,
&jnlseqno, &jnlseqno);
} else
send_msg_csa(CSA_ARG(csa) VARLSTCNT(5) ERR_JNLCLOSED, 3, DB_LEN_STR(jpc->region), &csa->ti->curr_tn);
#ifdef VMS
/* We can get a jnl_file_lost before the file is even created, so locking is done only if the lock exist */
if (0 != csa->jnl->jnllsb->lockid)
{
was_lockid = TRUE;
status = gtm_enqw(EFN$C_ENF, LCK$K_EXMODE, csa->jnl->jnllsb, LCK$M_CONVERT | LCK$M_NODLCKBLK,
NULL, 0, NULL, 0, NULL, PSL$C_USER, 0);
if (SS$_NORMAL == status)
status = csa->jnl->jnllsb->cond;
}
jnl_file_close(jpc->region, FALSE, FALSE);
if (was_lockid)
{
if (SS$_NORMAL == status)
status = gtm_deq(csa->jnl->jnllsb->lockid, NULL, PSL$C_USER, 0);
assertpro(SS$_NORMAL == status);
}
# else
jnl_file_close(jpc->region, FALSE, FALSE);
#endif
return EXIT_NRM;
}
* sent across to the update process that does not care about these fields so it is ok to leave them as is.
*/
jpc = csa->jnl;
assert((0 != jpc->pini_addr) || REPL_WAS_ENABLED(csa));
assert(jgbl.gbl_jrec_time || REPL_WAS_ENABLED(csa));
assert(csa->now_crit);
assert(IS_SET_KILL_ZKILL_ZTRIG_ZTWORM(jfb->rectype) || (JRT_NULL == jfb->rectype));
assert(!IS_ZTP(jfb->rectype));
jrec = (struct_jrec_upd *)jfb->buff;
assert(OFFSETOF(struct_jrec_null, prefix) == OFFSETOF(struct_jrec_upd, prefix));
assert(SIZEOF(jrec_null->prefix) == SIZEOF(jrec->prefix));
jrec->prefix.pini_addr = (0 == jpc->pini_addr) ? JNL_HDR_LEN : jpc->pini_addr;
jrec->prefix.tn = csa->ti->curr_tn;
jrec->prefix.time = jgbl.gbl_jrec_time;
/* t_end/tp_tend/mur_output_record has already set token/jnl_seqno into jnl_fence_ctl.token */
assert((0 != jnl_fence_ctl.token) || (!dollar_tlevel && !jgbl.forw_phase_recovery && !REPL_ENABLED(csa))
|| (!dollar_tlevel && jgbl.forw_phase_recovery && (repl_open != csa->hdr->intrpt_recov_repl_state)));
assert(OFFSETOF(struct_jrec_null, jnl_seqno) == OFFSETOF(struct_jrec_upd, token_seq));
assert(SIZEOF(jrec_null->jnl_seqno) == SIZEOF(jrec->token_seq));
jrec->token_seq.token = jnl_fence_ctl.token;
assert(OFFSETOF(struct_jrec_null, strm_seqno) == OFFSETOF(struct_jrec_upd, strm_seqno));
assert(SIZEOF(jrec_null->strm_seqno) == SIZEOF(jrec->strm_seqno));
jrec->strm_seqno = jnl_fence_ctl.strm_seqno;
/*update checksum below*/
if(JRT_NULL != jrec->prefix.jrec_type)
{
COMPUTE_LOGICAL_REC_CHECKSUM(jfb->checksum, jrec, com_csum, jrec->prefix.checksum);
}
else
jrec->prefix.checksum = compute_checksum(INIT_CHECKSUM_SEED, (uint4 *)jrec, SIZEOF(struct_jrec_null));
int gtmsource_checkhealth(void)
{
uint4 gtmsource_pid;
int status, semval, save_errno;
boolean_t srv_alive, all_files_open;
gtmsource_local_ptr_t gtmsourcelocal_ptr;
int4 index, num_servers;
seq_num reg_seqno, jnlseqno;
gd_region *reg, *region_top;
sgmnt_addrs *csa;
sgmnt_data_ptr_t csd;
char errtxt[OUT_BUFF_SIZE];
char *modestr;
assert(holds_sem[SOURCE][JNL_POOL_ACCESS_SEM]);
if (NULL != jnlpool.gtmsource_local) /* Check health of a specific source server */
gtmsourcelocal_ptr = jnlpool.gtmsource_local;
else
gtmsourcelocal_ptr = &jnlpool.gtmsource_local_array[0];
num_servers = 0;
status = SRV_ALIVE;
for (index = 0; index < NUM_GTMSRC_LCL; index++, gtmsourcelocal_ptr++)
{
if ('\0' == gtmsourcelocal_ptr->secondary_instname[0])
{
assert(NULL == jnlpool.gtmsource_local);
continue;
}
gtmsource_pid = gtmsourcelocal_ptr->gtmsource_pid;
/* If CHECKHEALTH on a specific secondary instance is requested, print the health information irrespective
* of whether a source server for that instance is alive or not. For CHECKHEALTH on ALL secondary instances
* print health information only for those instances that have an active or passive source server alive.
*/
if ((NULL == jnlpool.gtmsource_local) && (0 == gtmsource_pid))
continue;
repl_log(stdout, TRUE, TRUE, "Initiating CHECKHEALTH operation on source server pid [%d] for secondary instance"
" name [%s]\n", gtmsource_pid, gtmsourcelocal_ptr->secondary_instname);
srv_alive = (0 == gtmsource_pid) ? FALSE : is_proc_alive(gtmsource_pid, 0);
if (srv_alive)
{
if (GTMSOURCE_MODE_ACTIVE == gtmsourcelocal_ptr->mode)
modestr = "ACTIVE";
else if (GTMSOURCE_MODE_ACTIVE_REQUESTED == gtmsourcelocal_ptr->mode)
modestr = "ACTIVE REQUESTED";
else if (GTMSOURCE_MODE_PASSIVE == gtmsourcelocal_ptr->mode)
modestr = "PASSIVE";
else if (GTMSOURCE_MODE_PASSIVE_REQUESTED == gtmsourcelocal_ptr->mode)
modestr = "PASSIVE REQUESTED";
else
{
assert(gtmsourcelocal_ptr->mode != gtmsourcelocal_ptr->mode);
modestr = "UNKNOWN";
}
repl_log(stderr, FALSE, TRUE, FORMAT_STR1, gtmsource_pid, "Source server", "", modestr);
status |= SRV_ALIVE;
num_servers++;
} else
{
repl_log(stderr, FALSE, TRUE, FORMAT_STR, gtmsource_pid, "Source server", " NOT");
gtm_putmsg_csa(CSA_ARG(NULL) VARLSTCNT(4) ERR_SRCSRVNOTEXIST, 2,
LEN_AND_STR(gtmsourcelocal_ptr->secondary_instname));
status |= SRV_DEAD;
}
if (NULL != jnlpool.gtmsource_local)
break;
}
if (NULL == jnlpool.gtmsource_local)
{ /* Compare number of servers that were found alive with the current value of the COUNT semaphore.
* If they are not equal, report the discrepancy.
*/
semval = get_sem_info(SOURCE, SRC_SERV_COUNT_SEM, SEM_INFO_VAL);
if (-1 == semval)
{
save_errno = errno;
repl_log(stderr, FALSE, TRUE,
"Error fetching source server count semaphore value : %s\n", STRERROR(save_errno));
status |= SRV_ERR;
} else if (semval != num_servers)
{
repl_log(stderr, FALSE, FALSE,
"Error : Expected %d source server(s) to be alive but found %d actually alive\n",
semval, num_servers);
repl_log(stderr, FALSE, TRUE, "Error : Check if any pid reported above is NOT a source server process\n");
status |= SRV_ERR;
}
}
/* Check that there are no regions with replication state = WAS_ON (i.e. repl_was_open). If so report that.
* But to determine that, we need to attach to all the database regions.
*/
gvinit();
/* We use the same code dse uses to open all regions but we must make sure they are all open before proceeding. */
all_files_open = region_init(FALSE);
if (!all_files_open)
{
gtm_putmsg_csa(CSA_ARG(NULL) VARLSTCNT(1) ERR_NOTALLDBOPN);
status |= SRV_ERR;
} else
{
for (reg = gd_header->regions, region_top = gd_header->regions + gd_header->n_regions; reg < region_top; reg++)
{
csa = &FILE_INFO(reg)->s_addrs;
csd = csa->hdr;
if (REPL_WAS_ENABLED(csd))
{
assert(!JNL_ENABLED(csd) || REPL_ENABLED(csd)); /* || is for turning replication on concurrently */
reg_seqno = csd->reg_seqno;
jnlseqno = (NULL != jnlpool.jnlpool_ctl) ? jnlpool.jnlpool_ctl->jnl_seqno : MAX_SEQNO;
sgtm_putmsg(errtxt, VARLSTCNT(8) ERR_REPLJNLCLOSED, 6, DB_LEN_STR(reg),
®_seqno, ®_seqno, &jnlseqno, &jnlseqno);
repl_log(stderr, FALSE, TRUE, errtxt);
status |= SRV_ERR;
}
}
}
if (jnlpool.jnlpool_ctl->freeze)
{
repl_log(stderr, FALSE, FALSE, "Warning: Instance Freeze is ON\n");
repl_log(stderr, FALSE, TRUE, " Freeze Comment: %s\n", jnlpool.jnlpool_ctl->freeze_comment);
status |= SRV_ERR;
}
return (status + NORMAL_SHUTDOWN);
}
/* This called for TP and non-TP, but not for ZTP */
void jnl_write_logical(sgmnt_addrs *csa, jnl_format_buffer *jfb, uint4 com_csum, jnlpool_write_ctx_t *jplctx)
{
struct_jrec_upd *jrec;
struct_jrec_null *jrec_null;
struct_jrec_upd *jrec_alt;
jnl_private_control *jpc;
/* If REPL_WAS_ENABLED(csa) is TRUE, then we would not have gone through the code that initializes
* jgbl.gbl_jrec_time or jpc->pini_addr. But in this case, we are not writing the journal record
* to the journal buffer or journal file but write it only to the journal pool from where it gets
* sent across to the update process that does not care about these fields so it is ok to leave them as is.
*/
jpc = csa->jnl;
assert((0 != jpc->pini_addr) || REPL_WAS_ENABLED(csa));
assert(jgbl.gbl_jrec_time || REPL_WAS_ENABLED(csa));
assert(csa->now_crit);
assert(IS_SET_KILL_ZKILL_ZTWORM_LGTRIG_ZTRIG(jfb->rectype) || (JRT_NULL == jfb->rectype));
assert(!IS_ZTP(jfb->rectype));
jrec = (struct_jrec_upd *)jfb->buff;
assert(OFFSETOF(struct_jrec_null, prefix) == OFFSETOF(struct_jrec_upd, prefix));
assert(SIZEOF(jrec_null->prefix) == SIZEOF(jrec->prefix));
jrec->prefix.pini_addr = (0 == jpc->pini_addr) ? JNL_HDR_LEN : jpc->pini_addr;
jrec->prefix.tn = csa->ti->curr_tn;
jrec->prefix.time = jgbl.gbl_jrec_time;
/* t_end/tp_tend/mur_output_record has already set token/jnl_seqno into jnl_fence_ctl.token */
assert((0 != jnl_fence_ctl.token) || (!dollar_tlevel && !jgbl.forw_phase_recovery && !REPL_ENABLED(csa))
|| (!dollar_tlevel && jgbl.forw_phase_recovery && (repl_open != csa->hdr->intrpt_recov_repl_state)));
assert(OFFSETOF(struct_jrec_null, jnl_seqno) == OFFSETOF(struct_jrec_upd, token_seq));
assert(SIZEOF(jrec_null->jnl_seqno) == SIZEOF(jrec->token_seq));
jrec->token_seq.token = jnl_fence_ctl.token;
assert(OFFSETOF(struct_jrec_null, strm_seqno) == OFFSETOF(struct_jrec_upd, strm_seqno));
assert(SIZEOF(jrec_null->strm_seqno) == SIZEOF(jrec->strm_seqno));
jrec->strm_seqno = jnl_fence_ctl.strm_seqno;
/* update checksum below */
if(JRT_NULL != jrec->prefix.jrec_type)
{
COMPUTE_LOGICAL_REC_CHECKSUM(jfb->checksum, jrec, com_csum, jrec->prefix.checksum);
} else
jrec->prefix.checksum = compute_checksum(INIT_CHECKSUM_SEED, (unsigned char *)jrec, SIZEOF(struct_jrec_null));
if (REPL_ALLOWED(csa) && USES_ANY_KEY(csa->hdr))
{
jrec_alt = (struct_jrec_upd *)jfb->alt_buff;
jrec_alt->prefix = jrec->prefix;
jrec_alt->token_seq = jrec->token_seq;
jrec_alt->strm_seqno = jrec->strm_seqno;
jrec_alt->num_participants = jrec->num_participants;
}
JNL_WRITE_APPROPRIATE(csa, jpc, jfb->rectype, (jnl_record *)jrec, NULL, jfb, jplctx);
}
void mutex_deadlock_check(mutex_struct_ptr_t criticalPtr)
{
tp_region *tr;
sgmnt_addrs *csa;
int4 save_crit_count;
if (in_mutex_deadlock_check)
return;
in_mutex_deadlock_check = TRUE;
/* A zero value of "crit_count" implies asynchronous activities can occur (e.g. db flush timer, periodic epoch timers etc.).
* At this point, although we are here through grab_crit()/grab_lock() (which would have incremented "crit_count"), we are
* in a safe and consistent state as far as the mutex structures go so it is ok to set "crit_count" to 0 implying we
* are now in an interruptible state (of course, we need to restore "crit_count" to what it was before returning).
* The other alternative of not changing "crit_count" presents us with complex situations wherein recursion
* of grab_crit/rel_crit might occur (through direct or indirect calls from mutex_deadlock_check())
* causing crit_count to be > 1 and in turn causing the crit_count-reset-logic in grab_crit/rel_crit to
* do a "crit_count--" (instead of "crit_count = 0"). This suffers from the problem that in case of an error code path
* crit_count might not get decremented appropriately and hence become out-of-sync (i.e. a positive value instead
* of zero) and a non-zero value might cause indefinite deferrals of asynchronous events.
*/
assert(1 == crit_count);
save_crit_count = crit_count;
crit_count = 0;
/* Need to determine who should and should not go through the deadlock checker.
*
* List of who needs to be considered
* ------------------------------------
* -> GT.M, Update process, MUPIP LOAD and GT.CM GNP/OMI server : since they go through t_end() to update the database.
* Note that all of the above (and only those) have the "is_replicator" flag set to TRUE.
* -> MUPIP REORG, since it does non-TP transactions and goes through t_end() (has "mu_reorg_process" flag set).
*
* List of who does not need to be considered (with reasons)
* -----------------------------------------------------------
* -> MUPIP RECOVER can hold crit on several regions (through TP or non-TP transactions).
* But it has standalone access and hence no possibility of a deadlock.
* -> MUPIP RESTORE too holds standalone access so does not need to be considered.
* -> Source Server, Receiver Server etc. can hold only one CRIT resource at any point of time.
* -> DSE, MUPIP BACKUP, MUPIP SET JOURNAL etc. can legitimately hold crit on several regions though in non-TP.
*/
if (is_replicator || mu_reorg_process)
{
if (0 == dollar_tlevel)
{
if ((NULL != jnlpool.jnlpool_dummy_reg) && jnlpool.jnlpool_dummy_reg->open)
{
++crit_deadlock_check_cycle;
if (FILE_INFO(jnlpool.jnlpool_dummy_reg)->s_addrs.critical == criticalPtr)
{ /* grab_lock going for crit on the jnlpool region. gv_cur_region points to the
* current region of interest, which better have replication enabled, and be now crit
*/
assert(cs_addrs == &FILE_INFO(gv_cur_region)->s_addrs);
csa = &FILE_INFO(gv_cur_region)->s_addrs;
if (FALSE == csa->now_crit || !REPL_ENABLED(csa->hdr))
GTMASSERT; /* should have crit on gv_cur_region before asking for jnlpool */
csa->crit_check_cycle = crit_deadlock_check_cycle; /* allow for crit in gv_cur_region */
}
}
} else
{ /* Need to mark the regions allowed to have crit as follows:
* Place the current cycle into the csa's of regions allowed to have crit so have_crit() can easily test.
* Note that should the system be up long enough for the 2**32 cycle value to
* wrap and a region be unused for most of that time, such a region might not be entitled to crit
* but have an old csa->crit_cycle_check matching the current crit_deadlock_cycle_check -
* that case would not trigger have_crit() to release crit on that region;
* however, the next call to this routine increments crit_deadlock_check_cycle and so
* crit on that region gets released after two calls instead of (the usual) one.
*/
++crit_deadlock_check_cycle;
for (tr = tp_reg_list; NULL != tr; tr = tr->fPtr)
{
if (!tr->reg->open)
continue;
csa = &FILE_INFO(tr->reg)->s_addrs;
if (csa->now_crit)
csa->crit_check_cycle = crit_deadlock_check_cycle;
else
{ /* Seen first non-crit region. Make sure either of the following is true.
* (i) this is the region we are currently grabbing crit on
* (ii) we do not hold crit on any region in the tp_reg_list.
* If neither of the above, we have an out of design condition that can only
* warrant blowing the process up..
*/
if ((csa->critical != criticalPtr) && (tr != tp_reg_list))
GTMASSERT;
break;
}
}
}
/* Release crit in regions not legitimately part of this TP/non-TP transaction */
have_crit(CRIT_RELEASE | CRIT_NOT_TRANS_REG);
}
crit_count = save_crit_count;
in_mutex_deadlock_check = FALSE;
}
