/* Remove a realserver IPVS rule */
static int
clear_service_rs(virtual_server_t * vs, list l)
{
element e;
real_server_t *rs;
long weight_sum;
long down_threshold = vs->quorum - vs->hysteresis;
for (e = LIST_HEAD(l); e; ELEMENT_NEXT(e)) {
rs = ELEMENT_DATA(e);
if (ISALIVE(rs)) {
log_message(LOG_INFO, "Removing service %s from VS %s"
, FMT_RS(rs)
, FMT_VS(vs));
if (!ipvs_cmd(LVS_CMD_DEL_DEST, vs, rs))
return 0;
UNSET_ALIVE(rs);
if (!vs->omega)
continue;
/* In Omega mode we call VS and RS down notifiers
* all the way down the exit, as necessary.
*/
if (rs->notify_down) {
log_message(LOG_INFO, "Executing [%s] for service %s in VS %s"
, rs->notify_down
, FMT_RS(rs)
, FMT_VS(vs));
notify_exec(rs->notify_down);
}
#ifdef _WITH_SNMP_
check_snmp_rs_trap(rs, vs);
#endif
/* Sooner or later VS will lose the quorum (if any). However,
* we don't push in a sorry server then, hence the regression
* is intended.
*/
weight_sum = weigh_live_realservers(vs);
if (vs->quorum_state == UP && (
!weight_sum ||
weight_sum < down_threshold)
) {
vs->quorum_state = DOWN;
if (vs->quorum_down) {
log_message(LOG_INFO, "Executing [%s] for VS %s"
, vs->quorum_down
, FMT_VS(vs));
notify_exec(vs->quorum_down);
}
#ifdef _WITH_SNMP_
check_snmp_quorum_trap(vs);
#endif
}
}
}
return 1;
}
/* Remove a realserver IPVS rule */
static int
clear_service_rs(list vs_group, virtual_server * vs, list l)
{
element e;
real_server *rs;
char rsip[16], vsip[16];
for (e = LIST_HEAD(l); e; ELEMENT_NEXT(e)) {
rs = ELEMENT_DATA(e);
if (ISALIVE(rs)) {
if (!ipvs_cmd(LVS_CMD_DEL_DEST
, vs_group
, vs
, rs))
return 0;
UNSET_ALIVE(rs);
if (!vs->omega)
continue;
/* In Omega mode we call VS and RS down notifiers
* all the way down the exit, as necessary.
*/
if (rs->notify_down) {
log_message(LOG_INFO, "Executing [%s] for service [%s:%d]"
" in VS [%s:%d]"
, rs->notify_down
, inet_ntoa2(SVR_IP(rs), rsip)
, ntohs(SVR_PORT(rs))
, (vs->vsgname) ? vs->vsgname : inet_ntoa2(SVR_IP(vs), vsip)
, ntohs(SVR_PORT(vs)));
notify_exec(rs->notify_down);
}
/* Sooner or later VS will lose the quorum (if any). However,
* we don't push in a sorry server then, hence the regression
* is intended.
*/
if (vs->quorum_state == UP && vs->quorum_down
&& weigh_live_realservers(vs) < vs->quorum - vs->hysteresis) {
vs->quorum_state = DOWN;
log_message(LOG_INFO, "Executing [%s] for VS [%s:%d]"
, vs->quorum_down
, (vs->vsgname) ? vs->vsgname : inet_ntoa2(SVR_IP(vs), vsip)
, ntohs(SVR_PORT(vs)));
notify_exec(vs->quorum_down);
}
}
#ifdef _KRNL_2_2_
/* if we have a /32 mask, we create one nat rules per
* realserver.
*/
if (vs->nat_mask == HOST_NETMASK)
if (!ipfw_cmd(IP_FW_CMD_DEL, vs, rs))
return 0;
#endif
}
return 1;
}
/* Remove a realserver IPVS rule */
static int
clear_service_rs(list vs_group, virtual_server * vs, list l)
{
element e;
real_server *rs;
char rsip[INET6_ADDRSTRLEN];
for (e = LIST_HEAD(l); e; ELEMENT_NEXT(e)) {
rs = ELEMENT_DATA(e);
if (ISALIVE(rs)) {
if (!ipvs_cmd(LVS_CMD_DEL_DEST, vs_group, vs, rs))
return 0;
UNSET_ALIVE(rs);
if (!vs->omega)
continue;
/* In Omega mode we call VS and RS down notifiers
* all the way down the exit, as necessary.
*/
if (rs->notify_down) {
log_message(LOG_INFO, "Executing [%s] for service [%s]:%d in VS [%s]:%d"
, rs->notify_down
, inet_sockaddrtos2(&rs->addr, rsip)
, ntohs(inet_sockaddrport(&rs->addr))
, (vs->vsgname) ? vs->vsgname : inet_sockaddrtos(&vs->addr)
, ntohs(inet_sockaddrport(&vs->addr)));
notify_exec(rs->notify_down);
}
/* Sooner or later VS will lose the quorum (if any). However,
* we don't push in a sorry server then, hence the regression
* is intended.
*/
if (vs->quorum_state == UP && vs->quorum_down &&
weigh_live_realservers(vs) < vs->quorum - vs->hysteresis) {
vs->quorum_state = DOWN;
log_message(LOG_INFO, "Executing [%s] for VS [%s]:%d"
, vs->quorum_down
, (vs->vsgname) ? vs->vsgname : inet_sockaddrtos(&vs->addr)
, ntohs(inet_sockaddrport(&vs->addr)));
notify_exec(vs->quorum_down);
}
}
}
return 1;
}
/* set quorum state depending on current weight of real servers */
static void
update_quorum_state(virtual_server_t * vs)
{
long weight_sum = weigh_live_realservers(vs);
long up_threshold = vs->quorum + vs->hysteresis;
long down_threshold = vs->quorum - vs->hysteresis;
/* If we have just gained quorum, it's time to consider notify_up. */
if (vs->quorum_state == DOWN &&
weight_sum >= up_threshold) {
vs->quorum_state = UP;
log_message(LOG_INFO, "Gained quorum %lu+%lu=%li <= %li for VS %s"
, vs->quorum
, vs->hysteresis
, up_threshold
, weight_sum
, FMT_VS(vs));
if (vs->s_svr && ISALIVE(vs->s_svr)) {
log_message(LOG_INFO, "%s sorry server %s from VS %s"
, (vs->s_svr->inhibit ? "Disabling" : "Removing")
, FMT_RS(vs->s_svr)
, FMT_VS(vs));
ipvs_cmd(LVS_CMD_DEL_DEST, vs, vs->s_svr);
vs->s_svr->alive = 0;
/* Adding back alive real servers */
perform_quorum_state(vs, 1);
}
if (vs->quorum_up) {
log_message(LOG_INFO, "Executing [%s] for VS %s"
, vs->quorum_up
, FMT_VS(vs));
notify_exec(vs->quorum_up);
}
#ifdef _WITH_SNMP_CHECKER_
check_snmp_quorum_trap(vs);
#endif
return;
}
/* If we have just lost quorum for the VS, we need to consider
* VS notify_down and sorry_server cases
*/
if (vs->quorum_state == UP &&
(!weight_sum || weight_sum < down_threshold)
) {
vs->quorum_state = DOWN;
log_message(LOG_INFO, "Lost quorum %lu-%lu=%li > %li for VS %s"
, vs->quorum
, vs->hysteresis
, down_threshold
, weight_sum
, FMT_VS(vs));
if (vs->quorum_down) {
log_message(LOG_INFO, "Executing [%s] for VS %s"
, vs->quorum_down
, FMT_VS(vs));
notify_exec(vs->quorum_down);
}
if (vs->s_svr) {
log_message(LOG_INFO, "%s sorry server %s to VS %s"
, (vs->s_svr->inhibit ? "Enabling" : "Adding")
, FMT_RS(vs->s_svr)
, FMT_VS(vs));
/* the sorry server is now up in the pool, we flag it alive */
ipvs_cmd(LVS_CMD_ADD_DEST, vs, vs->s_svr);
vs->s_svr->alive = 1;
/* Remove remaining alive real servers */
perform_quorum_state(vs, 0);
}
#ifdef _WITH_SNMP_CHECKER_
check_snmp_quorum_trap(vs);
#endif
return;
}
}
/* set quorum state depending on current weight of real servers */
void
update_quorum_state(virtual_server * vs)
{
char rsip[INET6_ADDRSTRLEN];
/* If we have just gained quorum, it's time to consider notify_up. */
if (vs->quorum_state == DOWN &&
weigh_live_realservers(vs) >= vs->quorum + vs->hysteresis) {
vs->quorum_state = UP;
log_message(LOG_INFO, "Gained quorum %lu+%lu=%lu <= %u for VS [%s]:%d"
, vs->quorum
, vs->hysteresis
, vs->quorum + vs->hysteresis
, weigh_live_realservers(vs)
, (vs->vsgname) ? vs->vsgname : inet_sockaddrtos(&vs->addr)
, ntohs(inet_sockaddrport(&vs->addr)));
if (vs->s_svr && ISALIVE(vs->s_svr)) {
log_message(LOG_INFO, "Removing sorry server [%s]:%d from VS [%s]:%d"
, inet_sockaddrtos2(&vs->s_svr->addr, rsip)
, ntohs(inet_sockaddrport(&vs->s_svr->addr))
, (vs->vsgname) ? vs->vsgname : inet_sockaddrtos(&vs->addr)
, ntohs(inet_sockaddrport(&vs->addr)));
ipvs_cmd(LVS_CMD_DEL_DEST, check_data->vs_group, vs, vs->s_svr);
vs->s_svr->alive = 0;
/* Adding back alive real servers */
perform_quorum_state(vs, 1);
}
if (vs->quorum_up) {
log_message(LOG_INFO, "Executing [%s] for VS [%s]:%d"
, vs->quorum_up
, (vs->vsgname) ? vs->vsgname : inet_sockaddrtos(&vs->addr)
, ntohs(inet_sockaddrport(&vs->addr)));
notify_exec(vs->quorum_up);
}
return;
}
/* If we have just lost quorum for the VS, we need to consider
* VS notify_down and sorry_server cases
*/
if (vs->quorum_state == UP &&
weigh_live_realservers(vs) < vs->quorum - vs->hysteresis) {
vs->quorum_state = DOWN;
log_message(LOG_INFO, "Lost quorum %lu-%lu=%lu > %u for VS [%s]:%d"
, vs->quorum
, vs->hysteresis
, vs->quorum - vs->hysteresis
, weigh_live_realservers(vs)
, (vs->vsgname) ? vs->vsgname : inet_sockaddrtos(&vs->addr)
, ntohs(inet_sockaddrport(&vs->addr)));
if (vs->quorum_down) {
log_message(LOG_INFO, "Executing [%s] for VS [%s]:%d"
, vs->quorum_down
, (vs->vsgname) ? vs->vsgname : inet_sockaddrtos(&vs->addr)
, ntohs(inet_sockaddrport(&vs->addr)));
notify_exec(vs->quorum_down);
}
if (vs->s_svr) {
log_message(LOG_INFO, "Adding sorry server [%s]:%d to VS [%s]:%d"
, inet_sockaddrtos2(&vs->s_svr->addr, rsip)
, ntohs(inet_sockaddrport(&vs->s_svr->addr))
, (vs->vsgname) ? vs->vsgname : inet_sockaddrtos(&vs->addr)
, ntohs(inet_sockaddrport(&vs->addr)));
/* the sorry server is now up in the pool, we flag it alive */
ipvs_cmd(LVS_CMD_ADD_DEST, check_data->vs_group, vs, vs->s_svr);
vs->s_svr->alive = 1;
/* Remove remaining alive real servers */
perform_quorum_state(vs, 0);
}
return;
}
}
/* manipulate add/remove rs according to alive state */
void
perform_svr_state(int alive, virtual_server * vs, real_server * rs)
{
char rsip[16], vsip[16];
/*
* | ISALIVE(rs) | alive | context
* | 0 | 0 | first check failed under alpha mode, unreachable here
* | 0 | 1 | RS went up, add it to the pool
* | 1 | 0 | RS went down, remove it from the pool
* | 1 | 1 | first check succeeded w/o alpha mode, unreachable here
*/
if (!ISALIVE(rs) && alive) {
/* adding a server to the vs pool, if sorry server is flagged alive,
* we remove it from the vs pool.
*/
if (vs->s_svr) {
if (ISALIVE(vs->s_svr) &&
(vs->quorum_state == UP ||
(weigh_live_realservers(vs) + rs->weight >=
vs->quorum + vs->hysteresis))) {
log_message(LOG_INFO,
"Removing sorry server [%s:%d] from VS [%s:%d]",
inet_ntoa2(SVR_IP(vs->s_svr), rsip)
, ntohs(SVR_PORT(vs->s_svr))
, (vs->vsgname) ? vs->vsgname : inet_ntoa2(SVR_IP(vs), vsip)
, ntohs(SVR_PORT(vs)));
ipvs_cmd(LVS_CMD_DEL_DEST
, check_data->vs_group
, vs
, vs->s_svr);
vs->s_svr->alive = 0;
#ifdef _KRNL_2_2_
ipfw_cmd(IP_FW_CMD_DEL, vs, vs->s_svr);
#endif
}
}
log_message(LOG_INFO, "%s service [%s:%d] to VS [%s:%d]",
(rs->inhibit) ? "Enabling" : "Adding"
, inet_ntoa2(SVR_IP(rs), rsip)
, ntohs(SVR_PORT(rs))
, (vs->vsgname) ? vs->vsgname : inet_ntoa2(SVR_IP(vs), vsip)
, ntohs(SVR_PORT(vs)));
ipvs_cmd(LVS_CMD_ADD_DEST, check_data->vs_group, vs, rs);
rs->alive = alive;
if (rs->notify_up) {
log_message(LOG_INFO, "Executing [%s] for service [%s:%d]"
" in VS [%s:%d]"
, rs->notify_up
, inet_ntoa2(SVR_IP(rs), rsip)
, ntohs(SVR_PORT(rs))
, (vs->vsgname) ? vs->vsgname : inet_ntoa2(SVR_IP(vs), vsip)
, ntohs(SVR_PORT(vs)));
notify_exec(rs->notify_up);
}
/* If we have just gained quorum, it's time to consider notify_up. */
if (vs->quorum_state == DOWN
&& weigh_live_realservers(vs) >= vs->quorum + vs->hysteresis) {
vs->quorum_state = UP;
log_message(LOG_INFO, "Gained quorum %lu+%lu=%lu <= %u for VS [%s:%d]"
, vs->quorum
, vs->hysteresis
, vs->quorum + vs->hysteresis
, weigh_live_realservers(vs)
, (vs->vsgname) ? vs->vsgname : inet_ntoa2(SVR_IP(vs), vsip)
, ntohs(SVR_PORT(vs)));
if (vs->quorum_up) {
log_message(LOG_INFO, "Executing [%s] for VS [%s:%d]"
, vs->quorum_up
, (vs->vsgname) ? vs->vsgname : inet_ntoa2(SVR_IP(vs), vsip)
, ntohs(SVR_PORT(vs)));
notify_exec(vs->quorum_up);
}
}
#ifdef _KRNL_2_2_
if (vs->nat_mask == HOST_NETMASK)
ipfw_cmd(IP_FW_CMD_ADD, vs, rs);
#endif
return;
}
if (ISALIVE(rs) && !alive) {
log_message(LOG_INFO, "%s service [%s:%d] from VS [%s:%d]",
(rs->inhibit) ? "Disabling" : "Removing"
, inet_ntoa2(SVR_IP(rs), rsip)
, ntohs(SVR_PORT(rs))
, (vs->vsgname) ? vs->vsgname : inet_ntoa2(SVR_IP(vs), vsip)
, ntohs(SVR_PORT(vs)));
/* server is down, it is removed from the LVS realserver pool */
ipvs_cmd(LVS_CMD_DEL_DEST, check_data->vs_group, vs, rs);
rs->alive = alive;
if (rs->notify_down) {
log_message(LOG_INFO, "Executing [%s] for service [%s:%d]"
" in VS [%s:%d]"
, rs->notify_down
, inet_ntoa2(SVR_IP(rs), rsip)
, ntohs(SVR_PORT(rs))
, (vs->vsgname) ? vs->vsgname : inet_ntoa2(SVR_IP(vs), vsip)
, ntohs(SVR_PORT(vs)));
notify_exec(rs->notify_down);
}
#ifdef _KRNL_2_2_
if (vs->nat_mask == HOST_NETMASK)
ipfw_cmd(IP_FW_CMD_DEL, vs, rs);
#endif
/* If we have just lost quorum for the VS, we need to consider
* VS notify_down and sorry_server cases
*/
if (vs->quorum_state == UP
&& weigh_live_realservers(vs) < vs->quorum - vs->hysteresis) {
vs->quorum_state = DOWN;
log_message(LOG_INFO, "Lost quorum %lu-%lu=%lu > %u for VS [%s:%d]"
, vs->quorum
, vs->hysteresis
, vs->quorum - vs->hysteresis
, weigh_live_realservers(vs)
, (vs->vsgname) ? vs->vsgname : inet_ntoa2(SVR_IP(vs), vsip)
, ntohs(SVR_PORT(vs)));
if (vs->quorum_down) {
log_message(LOG_INFO, "Executing [%s] for VS [%s:%d]"
, vs->quorum_down
, (vs->vsgname) ? vs->vsgname : inet_ntoa2(SVR_IP(vs), vsip)
, ntohs(SVR_PORT(vs)));
notify_exec(vs->quorum_down);
}
if (vs->s_svr) {
log_message(LOG_INFO,
"Adding sorry server [%s:%d] to VS [%s:%d]",
inet_ntoa2(SVR_IP(vs->s_svr), rsip)
, ntohs(SVR_PORT(vs->s_svr))
, (vs->vsgname) ? vs->vsgname : inet_ntoa2(SVR_IP(vs), vsip)
, ntohs(SVR_PORT(vs)));
/* the sorry server is now up in the pool, we flag it alive */
ipvs_cmd(LVS_CMD_ADD_DEST, check_data->vs_group, vs, vs->s_svr);
vs->s_svr->alive = 1;
#ifdef _KRNL_2_2_
ipfw_cmd(IP_FW_CMD_ADD, vs, vs->s_svr);
#endif
}
}
}
}
