//------------------------------------------------------------------------------
// Add a new socket/port connection to this UM. It will be grouped with other
// socket/port connections having the same IP address for this UM.
// ioSock (in) - socket/port connection to be added
// writeLock (in) - mutex to use when writing to ioSock.
// return - boolean indicating whether socket/port connection was added.
//------------------------------------------------------------------------------
bool
UmModuleIPs::addSocketConn(
const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock )
{
bool bConnAdded = false;
boost::mutex::scoped_lock lock( fUmModuleMutex );
for (unsigned int i=0; i<fUmIPSocketConns.size(); ++i)
{
sockaddr sa = ioSock->sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
if (fUmIPSocketConns[i]->ipAddress() == sinp->sin_addr.s_addr)
{
#ifdef MOD_CONN_DEBUG
std::ostringstream oss;
oss << "UM " << fUmModuleName << "; adding connection " <<
ioSock->toString() << std::endl;
std::cout << oss.str();
#endif
fUmIPSocketConns[i]->addSocketConn ( ioSock, writeLock );
bConnAdded = true;
//..Initialize fNextUmIPSocketIdx if this is the first socket/port
// connection for this UM.
if ( fNextUmIPSocketIdx == NEXT_IP_SOCKET_UNASSIGNED)
fNextUmIPSocketIdx = i;
break;
}
}
return bConnAdded;
}
//------------------------------------------------------------------------------
// Add a new socket/port connection. It will be grouped with other
// socket/port connections belonging to the same UM.
// ios (in) - socket/port connection to be added
// writeLock (in) - mutex to use when writing to ios.
// return - boolean indicating whether socket/port connection was added.
//------------------------------------------------------------------------------
bool
UmSocketSelector::addConnection(
const SP_UM_IOSOCK& ios,
const SP_UM_MUTEX& writeLock )
{
bool bConnAdded = false;
sockaddr sa = ios->sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
IpAddressUmMap_t::iterator mapIter =
fIpAddressUmMap.find ( sinp->sin_addr.s_addr );
// Add this socket/port connection to the UM connection list it belongs to.
if ( mapIter != fIpAddressUmMap.end() )
{
unsigned int umIdx = mapIter->second;
bConnAdded = fUmModuleIPs[umIdx]->addSocketConn( ios, writeLock );
}
if (!bConnAdded)
{
#ifdef SEL_CONN_DEBUG
std::ostringstream oss;
oss << "No UM/IP match found to add connection " << ios->toString() <<
std::endl;
std::cout << oss.str();
#endif
}
return bConnAdded;
}
//------------------------------------------------------------------------------
// Get the next socket/port connection belonging to the same UM as ios.
// The selected socket/port connection will be returned in outIos. It can
// then be used for sending the next response message back to the applicable
// UM module.
// ios (in) - socket/port connection where a UM request originated from
// outIos (out) - socket/port connection to use in sending the
// corresponding response
// writelock (out) - mutex lock to be used when writing to outIos
// return - bool indicating if socket/port connection was assigned to
// outIos
//------------------------------------------------------------------------------
bool
UmSocketSelector::nextIOSocket(
const IOSocket& ios,
SP_UM_IOSOCK& outIos,
SP_UM_MUTEX& writeLock )
{
sockaddr sa = ios.sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
IpAddressUmMap_t::iterator mapIter =
fIpAddressUmMap.find ( sinp->sin_addr.s_addr );
if ( mapIter != fIpAddressUmMap.end() )
{
unsigned int umIdx = mapIter->second;
if (fUmModuleIPs[umIdx]->nextIOSocket( outIos, writeLock ))
{
#ifdef SEL_CONN_DEBUG
std::ostringstream oss;
oss << "UM " << fUmModuleIPs[umIdx]->moduleName() <<
"; in: " << ios.toString() <<
"; selected out: " << outIos->toString() << std::endl;
std::cout << oss.str();
#endif
return true;
}
}
//..This should not happen. Application is asking for next socket/port for
// a connection not in our UM module list.
return false;
}
int main()
{
// erydb.xml file should be configured as follows:
// um1: 10.100.4.85 and 10.100.5.85
// um2: 10.101.4.85 and 10.101.5.85
sockaddr_in sa = { 1, 0, {0} , {' '} };
char* ips[] = {"10.100.4.85", "10.100.5.85", "10.101.4.85", "10.101.5.85"};
// These are the IP addresses we use to test runtime connections
// "not" in the erydb.xml file.
sockaddr_in saUnknown = { 1, 0, {0} , {' '} };
char* ipsUnknown[]={"10.102.1.1", "10.102.2.1", "10.102.3.1", "10.102.4.1"};
//--------------------------------------------------------------------------
// Test initialization
//--------------------------------------------------------------------------
UmSocketSelector* sockSel = UmSocketSelector::instance();
std::cout << "IPAddressCount: " << sockSel->ipAddressCount() << std::endl;
std::cout << std::endl << "----Dump1 after initialization..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump1 End...................." << std::endl << std::endl;
IOSocket sock[4][4];
for (int i=0; i<4; i++)
{
inet_aton(ips[i], &sa.sin_addr);
for (int j=0; j<4; j++)
{
sock[i][j].setSocketImpl(new InetStreamSocket());
sa.sin_port = htons((i*4)+j);
sock[i][j].sa( sa );
sockSel->addConnection( SP_UM_IOSOCK(new IOSocket(sock[i][j])),
SP_UM_MUTEX( new boost::mutex()) );
}
}
std::cout << std::endl << "----Dump2 after adding 16 connections..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump2 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test socket/port selection
//--------------------------------------------------------------------------
std::cout << "Test socket/port selection logic..." << std::endl;
for (unsigned k=0; k<17; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
#if 1
if (sockSel->nextIOSocket( sock[0][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
#else
if (!sockSel->nextIOSocket( sock[0][0], outIos, writeLock ))
#endif
std::cout << "no nextIP found for " << sock[0][0] << std::endl;
}
for (unsigned k=0; k<7; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
if (sockSel->nextIOSocket( sock[2][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
std::cout << "no nextIP found for " << sock[2][0] << std::endl;
}
std::cout << std::endl;
std::cout << "----Dump3 after selecting 17 connections from IP " <<
ips[0] << "; and 7 connections from IP " << ips[2] << " ..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump3 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test connection deletions
//--------------------------------------------------------------------------
for (unsigned k=0; k<4; k++)
{
sockSel->delConnection( sock[k][0] );
}
std::cout << "----Dump4 after deleting first connection for each IP..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump4 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test addition of unknown connections
//--------------------------------------------------------------------------
IOSocket sockUnknown[4][4];
for (int i=0; i<4; i++)
{
inet_aton(ipsUnknown[i], &saUnknown.sin_addr);
for (int j=0; j<4; j++)
{
sockUnknown[i][j].setSocketImpl(new InetStreamSocket());
saUnknown.sin_port = htons((i*4)+j);
sockUnknown[i][j].sa( saUnknown );
sockSel->addConnection(
SP_UM_IOSOCK(new IOSocket(sockUnknown[i][j])),
SP_UM_MUTEX( new boost::mutex()) );
}
}
std::cout << "----Dump5 after adding connections for unknown IP's..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump5 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test resetting of "next" indexes after deleting all sockets from a
// specific IP address for which the "next" index is pointing.
//--------------------------------------------------------------------------
sockSel->delConnection( sock[1][1] );
sockSel->delConnection( sock[1][2] );
sockSel->delConnection( sock[1][3] );
std::cout << "----Dump6 after deleting all connections for IP " << ips[1] <<
" ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump6 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test socket/port selection for an unknown module
//--------------------------------------------------------------------------
std::cout << "Test socket/port selection logic..." << std::endl;
for (unsigned k=0; k<11; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
if (sockSel->nextIOSocket( sockUnknown[2][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
std::cout << "no nextIP found for " << sockUnknown[2][0]<<std::endl;
}
std::cout << std::endl;
std::cout << "----Dump7 after selecting 11 connections for IP " <<
ipsUnknown[2] << " ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump7 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test deletion of last socket/port connection and resetting if the
// "next" index that is pointing to it.
//--------------------------------------------------------------------------
sockSel->delConnection( sock[3][3] );
std::cout << "----Dump8 after deleting last connection for IP " <<
ips[3] << " ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump8 End...................." << std::endl << std::endl;
return 0;
}
void BPPSendThread::mainLoop()
{
const uint32_t msgCap = 20;
boost::scoped_array<Msg_t> msg;
uint32_t msgCount = 0, i, msgsSent;
SP_UM_MUTEX lock;
SP_UM_IOSOCK sock;
bool doLoadBalancing = false;
msg.reset(new Msg_t[msgCap]);
while (!die) {
mutex::scoped_lock sl(msgQueueLock);
if (msgQueue.empty() && !die) {
mainThreadWaiting = true;
queueNotEmpty.wait(sl);
mainThreadWaiting = false;
continue;
}
msgCount = (msgQueue.size() > msgCap ? msgCap : msgQueue.size());
for (i = 0; i < msgCount; i++) {
msg[i] = msgQueue.front();
msgQueue.pop();
}
doLoadBalancing = sawAllConnections;
sl.unlock();
/* In the send loop below, msgsSent tracks progress on sending the msg array,
* i how many msgs are sent by 1 run of the loop, limited by msgCount or msgsLeft. */
msgsSent = 0;
while (msgsSent < msgCount && !die) {
uint64_t bsSize;
if (msgsLeft <= 0 && fcEnabled && !die) {
mutex::scoped_lock sl2(ackLock);
while (msgsLeft <= 0 && fcEnabled && !die) {
waiting = true;
okToSend.wait(sl2);
waiting = false;
}
}
for (i = 0; msgsSent < msgCount && ((fcEnabled && msgsLeft > 0) || !fcEnabled) && !die;
msgsSent++, i++) {
if (doLoadBalancing) {
// Bug 4475 move control of sockIndex to batchPrimitiveProcessor
lock = connections_v[msg[msgsSent].sockIndex].sockLock;
sock = connections_v[msg[msgsSent].sockIndex].sock;
}
else {
lock = msg[msgsSent].sockLock;
sock = msg[msgsSent].sock;
}
bsSize = msg[msgsSent].msg->lengthWithHdrOverhead();
try {
mutex::scoped_lock sl2(*lock);
sock->write(*msg[msgsSent].msg);
//cout << "sent 1 msg\n";
}
catch (std::exception &e) {
sl.lock();
exceptionString = e.what();
gotException = true;
return;
}
(void)atomicops::atomicDec(&msgsLeft);
(void)atomicops::atomicSub(¤tByteSize, bsSize);
msg[msgsSent].msg.reset();
}
}
}
}
