void Slave::run()
{
string ip;
int port;
int32_t id;
SectorMsg* msg = new SectorMsg;
msg->resize(65536);
cout << "slave process: " << "GMP " << m_iLocalPort << " DATA " << m_DataChn.getPort() << endl;
while (true)
{
if (m_GMP.recvfrom(ip, port, id, msg) < 0)
break;
cout << "recv cmd " << ip << " " << port << " type " << msg->getType() << endl;
// a slave only accepts commands from the masters
Address addr;
addr.m_strIP = ip;
addr.m_iPort = port;
if (m_Routing.getRouterID(addr) < 0)
continue;
switch (msg->getType() / 100)
{
case 0:
processSysCmd(ip, port, id, msg);
break;
case 1:
processFSCmd(ip, port, id, msg);
break;
case 2:
processDCCmd(ip, port, id, msg);
break;
case 10:
processMCmd(ip, port, id, msg);
break;
default:
break;
}
}
delete msg;
}
int Slave::reportSphere(const string& master_ip, const int& master_port, const int& transid, const vector<Address>* bad)
{
SectorMsg msg;
msg.setType(4);
msg.setKey(0);
msg.setData(0, (char*)&transid, 4);
msg.setData(4, (char*)&m_iSlaveID, 4);
int num = (NULL == bad) ? 0 : bad->size();
msg.setData(8, (char*)&num, 4);
for (int i = 0; i < num; ++ i)
{
msg.setData(12 + 68 * i, (*bad)[i].m_strIP.c_str(), (*bad)[i].m_strIP.length() + 1);
msg.setData(12 + 68 * i + 64, (char*)&((*bad)[i].m_iPort), 4);
}
cout << "reportSphere " << master_ip << " " << master_port << " " << transid << endl;
if (m_GMP.rpc(master_ip.c_str(), master_port, &msg, &msg) < 0)
return -1;
if (msg.getType() < 0)
return *(int32_t*)msg.getData();
return 1;
}
int Client::list(const string& path, vector<SNode>& attr)
{
string revised_path = Metadata::revisePath(path);
SectorMsg msg;
msg.resize(65536);
msg.setType(101);
msg.setKey(m_iKey);
msg.setData(0, revised_path.c_str(), revised_path.length() + 1);
Address serv;
m_Routing.lookup(revised_path, serv);
login(serv.m_strIP, serv.m_iPort);
if (m_GMP.rpc(serv.m_strIP.c_str(), serv.m_iPort, &msg, &msg) < 0)
return SectorError::E_CONNECTION;
if (msg.getType() < 0)
return *(int32_t*)(msg.getData());
string filelist = msg.getData();
unsigned int s = 0;
while (s < filelist.length())
{
int t = filelist.find(';', s);
SNode sn;
sn.deserialize(filelist.substr(s, t - s).c_str());
attr.insert(attr.end(), sn);
s = t + 1;
}
return attr.size();
}
int Client::move(const string& oldpath, const string& newpath)
{
string src = Metadata::revisePath(oldpath);
string dst = Metadata::revisePath(newpath);
SectorMsg msg;
msg.setType(104);
msg.setKey(m_iKey);
int32_t size = src.length() + 1;
msg.setData(0, (char*)&size, 4);
msg.setData(4, src.c_str(), src.length() + 1);
size = dst.length() + 1;
msg.setData(4 + src.length() + 1, (char*)&size, 4);
msg.setData(4 + src.length() + 1 + 4, dst.c_str(), dst.length() + 1);
Address serv;
m_Routing.lookup(src, serv);
login(serv.m_strIP, serv.m_iPort);
if (m_GMP.rpc(serv.m_strIP.c_str(), serv.m_iPort, &msg, &msg) < 0)
return SectorError::E_CONNECTION;
if (msg.getType() < 0)
return *(int32_t*)(msg.getData());
return 0;
}
int Client::copy(const string& src, const string& dst)
{
string rsrc = Metadata::revisePath(src);
string rdst = Metadata::revisePath(dst);
SectorMsg msg;
msg.setType(106);
msg.setKey(m_iKey);
int32_t size = rsrc.length() + 1;
msg.setData(0, (char*)&size, 4);
msg.setData(4, rsrc.c_str(), rsrc.length() + 1);
size = rdst.length() + 1;
msg.setData(4 + rsrc.length() + 1, (char*)&size, 4);
msg.setData(4 + rsrc.length() + 1 + 4, rdst.c_str(), rdst.length() + 1);
Address serv;
m_Routing.lookup(rsrc, serv);
login(serv.m_strIP, serv.m_iPort);
if (m_GMP.rpc(serv.m_strIP.c_str(), serv.m_iPort, &msg, &msg) < 0)
return SectorError::E_CONNECTION;
if (msg.getType() < 0)
return *(int32_t*)(msg.getData());
return 0;
}
int Client::sysinfo(SysStat& sys)
{
SectorMsg msg;
msg.setKey(m_iKey);
msg.setType(3);
msg.m_iDataLength = SectorMsg::m_iHdrSize;
Address serv;
m_Routing.lookup(m_iKey, serv);
login(serv.m_strIP, serv.m_iPort);
if (m_GMP.rpc(serv.m_strIP.c_str(), serv.m_iPort, &msg, &msg) < 0)
return SectorError::E_CONNECTION;
if (msg.getType() < 0)
return *(int32_t*)(msg.getData());
deserializeSysStat(sys, msg.getData(), msg.m_iDataLength);
for (vector<Address>::iterator i = sys.m_vMasterList.begin(); i != sys.m_vMasterList.end(); ++ i)
{
if (i->m_strIP.length() == 0)
{
i->m_strIP = serv.m_strIP;
break;
}
}
return 0;
}
int Slave::report(const string& master_ip, const int& master_port, const int32_t& transid, const vector<string>& filelist, const int& change)
{
vector<string> serlist;
for (vector<string>::const_iterator i = filelist.begin(); i != filelist.end(); ++ i)
{
struct stat s;
if (-1 == stat((m_strHomeDir + *i).c_str(), &s))
continue;
SNode sn;
sn.m_strName = *i;
sn.m_bIsDir = S_ISDIR(s.st_mode) ? 1 : 0;
sn.m_llTimeStamp = s.st_mtime;
sn.m_llSize = s.st_size;
char buf[1024];
sn.serialize(buf);
//update local
Address addr;
addr.m_strIP = "127.0.0.1";
addr.m_iPort = 0;
m_pLocalFile->update(buf, addr, change);
serlist.push_back(buf);
}
if (serlist.empty())
return 0;
SectorMsg msg;
msg.setType(1);
msg.setKey(0);
msg.setData(0, (char*)&transid, 4);
msg.setData(4, (char*)&m_iSlaveID, 4);
msg.setData(8, (char*)&change, 4);
int32_t num = serlist.size();
msg.setData(12, (char*)&num, 4);
int pos = 16;
for (vector<string>::iterator i = serlist.begin(); i != serlist.end(); ++ i)
{
int32_t bufsize = i->length() + 1;
msg.setData(pos, (char*)&bufsize, 4);
msg.setData(pos + 4, i->c_str(), bufsize);
pos += bufsize + 4;
}
cout << "report " << master_ip << " " << master_port << " " << num << endl;
if (m_GMP.rpc(master_ip.c_str(), master_port, &msg, &msg) < 0)
return -1;
if (msg.getType() < 0)
return *(int32_t*)msg.getData();
return 1;
}
int FSClient::reopen()
{
if (0 == m_strFileName.length())
return -1;
// currently re-open only works on read
if (m_bWrite)
return -1;
// close connection to the current slave
int32_t cmd = 5;
m_pClient->m_DataChn.send(m_strSlaveIP, m_iSlaveDataPort, m_iSession, (char*)&cmd, 4);
int response;
m_pClient->m_DataChn.recv4(m_strSlaveIP, m_iSlaveDataPort, m_iSession, response);
m_pClient->m_DataChn.remove(m_strSlaveIP, m_iSlaveDataPort);
SectorMsg msg;
msg.setType(112); // open the file
msg.setKey(m_pClient->m_iKey);
msg.setData(0, (char*)&m_iSession, 4);
int32_t port = m_pClient->m_DataChn.getPort();
msg.setData(4, (char*)&port, 4);
Address serv;
m_pClient->lookup(m_strFileName, serv);
if (m_pClient->m_GMP.rpc(serv.m_strIP.c_str(), serv.m_iPort, &msg, &msg) < 0)
return SectorError::E_CONNECTION;
if (msg.getType() < 0)
return *(int32_t*)(msg.getData());
m_strSlaveIP = msg.getData();
m_iSlaveDataPort = *(int*)(msg.getData() + 64);
if (m_pClient->m_DataChn.connect(m_strSlaveIP, m_iSlaveDataPort) < 0)
return SectorError::E_CONNECTION;
memcpy(m_pcKey, m_pClient->m_pcCryptoKey, 16);
memcpy(m_pcIV, m_pClient->m_pcCryptoIV, 8);
m_pClient->m_DataChn.setCryptoKey(m_strSlaveIP, m_iSlaveDataPort, m_pcKey, m_pcIV);
return 0;
}
int Client::remove(const string& path)
{
string revised_path = Metadata::revisePath(path);
SectorMsg msg;
msg.setType(105);
msg.setKey(m_iKey);
msg.setData(0, revised_path.c_str(), revised_path.length() + 1);
Address serv;
m_Routing.lookup(revised_path, serv);
login(serv.m_strIP, serv.m_iPort);
if (m_GMP.rpc(serv.m_strIP.c_str(), serv.m_iPort, &msg, &msg) < 0)
return SectorError::E_CONNECTION;
if (msg.getType() < 0)
return *(int32_t*)(msg.getData());
return 0;
}
int DCClient::dataInfo(const vector<string>& files, vector<string>& info)
{
SectorMsg msg;
msg.setType(201);
msg.setKey(m_pClient->m_iKey);
int offset = 0;
int32_t size = -1;
for (vector<string>::const_iterator i = files.begin(); i != files.end(); ++ i)
{
string path = Metadata::revisePath(*i);
size = path.length() + 1;
msg.setData(offset, (char*)&size, 4);
msg.setData(offset + 4, path.c_str(), size);
offset += 4 + size;
}
size = -1;
msg.setData(offset, (char*)&size, 4);
Address serv;
m_pClient->m_Routing.getPrimaryMaster(serv);
if (m_pClient->m_GMP.rpc(serv.m_strIP.c_str(), serv.m_iPort, &msg, &msg) < 0)
return SectorError::E_CONNECTION;
if (msg.getType() < 0)
return *(int32_t*)(msg.getData());
char* buf = msg.getData();
size = msg.m_iDataLength - SectorMsg::m_iHdrSize;
while (size > 0)
{
info.insert(info.end(), buf);
size -= strlen(buf) + 1;
buf += strlen(buf) + 1;
}
return info.size();
}
int Client::stat(const string& path, SNode& attr)
{
string revised_path = Metadata::revisePath(path);
SectorMsg msg;
msg.resize(65536);
msg.setType(102);
msg.setKey(m_iKey);
msg.setData(0, revised_path.c_str(), revised_path.length() + 1);
Address serv;
m_Routing.lookup(revised_path, serv);
login(serv.m_strIP, serv.m_iPort);
if (m_GMP.rpc(serv.m_strIP.c_str(), serv.m_iPort, &msg, &msg) < 0)
return SectorError::E_CONNECTION;
if (msg.getType() < 0)
return *(int32_t*)(msg.getData());
attr.deserialize(msg.getData());
int n = (msg.m_iDataLength - SectorMsg::m_iHdrSize - 128) / 68;
char* al = msg.getData() + 128;
for (int i = 0; i < n; ++ i)
{
Address addr;
addr.m_strIP = al + 68 * i;
addr.m_iPort = *(int32_t*)(al + 68 * i + 64);
attr.m_sLocation.insert(addr);
}
// check local cache: updated files may not be sent to the master yet
m_StatCache.stat(path, attr);
return 0;
}
int FSClient::open(const string& filename, int mode, const string& hint)
{
m_strFileName = Metadata::revisePath(filename);
SectorMsg msg;
msg.setType(110); // open the file
msg.setKey(m_pClient->m_iKey);
int32_t m = mode;
msg.setData(0, (char*)&m, 4);
int32_t port = m_pClient->m_DataChn.getPort();
msg.setData(4, (char*)&port, 4);
msg.setData(8, hint.c_str(), hint.length() + 1);
msg.setData(72, m_strFileName.c_str(), m_strFileName.length() + 1);
Address serv;
m_pClient->m_Routing.lookup(m_strFileName, serv);
if (m_pClient->m_GMP.rpc(serv.m_strIP.c_str(), serv.m_iPort, &msg, &msg) < 0)
return SectorError::E_CONNECTION;
if (msg.getType() < 0)
return *(int32_t*)(msg.getData());
m_llSize = *(int64_t*)(msg.getData() + 72);
m_llCurReadPos = m_llCurWritePos = 0;
m_bRead = mode & 1;
m_bWrite = mode & 2;
m_bSecure = mode & 16;
// check APPEND
if (mode & 8)
m_llCurWritePos = m_llSize;
m_strSlaveIP = msg.getData();
m_iSlaveDataPort = *(int*)(msg.getData() + 64);
m_iSession = *(int*)(msg.getData() + 68);
cerr << "open file " << filename << " " << m_strSlaveIP << " " << m_iSlaveDataPort << endl;
if (m_pClient->m_DataChn.connect(m_strSlaveIP, m_iSlaveDataPort) < 0)
return SectorError::E_CONNECTION;
string localip;
int localport;
m_pClient->m_DataChn.getSelfAddr(m_strSlaveIP, m_iSlaveDataPort, localip, localport);
if (m_strSlaveIP == localip)
{
// the file is on the same node, check if the file can be read directly
int32_t cmd = 6;
m_pClient->m_DataChn.send(m_strSlaveIP, m_iSlaveDataPort, m_iSession, (char*)&cmd, 4);
int size = 0;
if (m_pClient->m_DataChn.recv(m_strSlaveIP, m_iSlaveDataPort, m_iSession, m_pcLocalPath, size) > 0)
{
fstream test((m_pcLocalPath + filename).c_str(), ios::binary | ios::in);
if (!test.bad() && !test.fail())
m_bLocal = true;
}
}
memcpy(m_pcKey, m_pClient->m_pcCryptoKey, 16);
memcpy(m_pcIV, m_pClient->m_pcCryptoIV, 8);
m_pClient->m_DataChn.setCryptoKey(m_strSlaveIP, m_iSlaveDataPort, m_pcKey, m_pcIV);
if (m_bWrite)
m_pClient->m_StatCache.insert(filename);
return 0;
}
int DCClient::connectSPE(SPE& s)
{
if (s.m_iStatus != 0)
return -1;
SectorMsg msg;
msg.setType(203); // start processing engine
msg.setKey(m_pClient->m_iKey);
msg.setData(0, s.m_strIP.c_str(), s.m_strIP.length() + 1);
msg.setData(64, (char*)&(s.m_iPort), 4);
// leave a 4-byte blank spot for data port
msg.setData(72, (char*)&(s.m_iID), 4);
msg.setData(76, (char*)&m_pClient->m_iKey, 4);
msg.setData(80, m_strOperator.c_str(), m_strOperator.length() + 1);
int offset = 80 + m_strOperator.length() + 1;
msg.setData(offset, (char*)&m_iRows, 4);
msg.setData(offset + 4, (char*)&m_iParamSize, 4);
msg.setData(offset + 8, m_pcParam, m_iParamSize);
offset += 4 + 8 + m_iParamSize;
msg.setData(offset, (char*)&m_iProcType, 4);
Address serv;
m_pClient->m_Routing.getPrimaryMaster(serv);
if ((m_pClient->m_GMP.rpc(serv.m_strIP.c_str(), serv.m_iPort, &msg, &msg) < 0) || (msg.getType() < 0))
return SectorError::E_CONNECTION;
s.m_iSession = *(int32_t*)msg.getData();
m_pClient->m_DataChn.connect(s.m_strIP, s.m_iDataPort);
cout << "connect SPE " << s.m_strIP.c_str() << " " << *(int*)(msg.getData()) << endl;
// send output information
m_pClient->m_DataChn.send(s.m_strIP, s.m_iDataPort, s.m_iSession, (char*)&m_iOutputType, 4);
if (m_iOutputType > 0)
{
int bnum = m_mBucket.size();
m_pClient->m_DataChn.send(s.m_strIP, s.m_iDataPort, s.m_iSession, (char*)&bnum, 4);
m_pClient->m_DataChn.send(s.m_strIP, s.m_iDataPort, s.m_iSession, m_pOutputLoc, bnum * 80);
m_pClient->m_DataChn.send(s.m_strIP, s.m_iDataPort, s.m_iSession, (char*)m_pOutput->m_piLocID, m_iOutputType * 4);
}
else if (m_iOutputType < 0)
m_pClient->m_DataChn.send(s.m_strIP, s.m_iDataPort, s.m_iSession, m_pOutputLoc, strlen(m_pOutputLoc) + 1);
loadOperator(s.m_strIP, s.m_iPort, s.m_iDataPort, s.m_iSession);
s.m_iStatus = 1;
return 0;
}
int DCClient::prepareOutput(const char* spenodes)
{
m_pOutputLoc = NULL;
m_pOutput->m_llSize = 0;
m_pOutput->m_llRecNum = 0;
// prepare output stream locations
if (m_iOutputType > 0)
{
SectorMsg msg;
msg.setType(204);
msg.setKey(m_pClient->m_iKey);
for (int i = 0; i < m_iSPENum; ++ i)
{
msg.setData(0, spenodes + i * 72, strlen(spenodes + i * 72) + 1);
msg.setData(64, spenodes + i * 72 + 64, 4);
msg.setData(68, (char*)&(m_pOutput->m_iFileNum), 4);
msg.setData(72, (char*)&i, 4);
int size = m_pOutput->m_strPath.length() + 1;
int offset = 76;
msg.setData(offset, (char*)&size, 4);
msg.setData(offset + 4, m_pOutput->m_strPath.c_str(), m_pOutput->m_strPath.length() + 1);
offset += 4 + size;
size = m_pOutput->m_strName.length() + 1;
msg.setData(offset, (char*)&size, 4);
msg.setData(offset + 4, m_pOutput->m_strName.c_str(), m_pOutput->m_strName.length() + 1);
offset += 4 + size;
msg.setData(offset, (char*)&m_pClient->m_iKey, 4);
offset += 4;
msg.setData(offset, (char*)&m_iProcType, 4);
if (m_iProcType == 1)
{
offset += 4;
size = m_strOperator.length() + 1;
msg.setData(offset, (char*)&size, 4);
msg.setData(offset + 4, m_strOperator.c_str(), m_strOperator.length() + 1);
}
cout << "request shuffler " << spenodes + i * 72 << " " << *(int*)(spenodes + i * 72 + 64) << endl;
Address serv;
m_pClient->m_Routing.getPrimaryMaster(serv);
if (m_pClient->m_GMP.rpc(serv.m_strIP.c_str(), serv.m_iPort, &msg, &msg) < 0)
continue;
if (msg.getType() < 0)
{
if (*(int32_t*)msg.getData() == SectorError::E_PERMISSION)
break;
else
continue;
}
BUCKET b;
b.m_iID = i;
b.m_strIP = spenodes + i * 72;
b.m_iPort = *(int32_t*)(spenodes + i * 72 + 64);
b.m_iDataPort = *(int32_t*)(spenodes + i * 72 + 68);
b.m_iShufflerPort = *(int32_t*)msg.getData();
b.m_iSession = *(int32_t*)(msg.getData() + 4);
b.m_iProgress = 0;
b.m_LastUpdateTime = CTimer::getTime();
m_mBucket[b.m_iID] = b;
// set up data connection, not for data transfter, but for keep-alive
m_pClient->m_DataChn.connect(b.m_strIP, b.m_iDataPort);
// upload library files for MapReduce processing
if (m_iProcType == 1)
loadOperator(b.m_strIP, b.m_iPort, b.m_iDataPort, b.m_iSession);
}
if (m_mBucket.empty())
return SectorError::E_NOBUCKET;
m_pOutputLoc = new char[m_mBucket.size() * 80];
int l = 0;
for (map<int, BUCKET>::iterator b = m_mBucket.begin(); b != m_mBucket.end(); ++ b)
{
strcpy(m_pOutputLoc + l * 80, b->second.m_strIP.c_str());
*(int32_t*)(m_pOutputLoc + l * 80 + 64) = b->second.m_iPort;
*(int32_t*)(m_pOutputLoc + l * 80 + 68) = b->second.m_iDataPort;
*(int32_t*)(m_pOutputLoc + l * 80 + 72) = b->second.m_iShufflerPort;
*(int32_t*)(m_pOutputLoc + l * 80 + 76) = b->second.m_iSession;
++ l;
}
// result locations
map<int, BUCKET>::iterator b = m_mBucket.begin();
for (int i = 0; i < m_pOutput->m_iFileNum; ++ i)
{
char* tmp = new char[m_pOutput->m_strPath.length() + m_pOutput->m_strName.length() + 64];
sprintf(tmp, "%s/%s.%d", m_pOutput->m_strPath.c_str(), m_pOutput->m_strName.c_str(), i);
m_pOutput->m_vFiles[i] = tmp;
delete [] tmp;
if (m_pOutput->m_vLocation[i].empty())
{
// if user didn't specify output location, simply pick the next bucket location and rotate
// this should be the normal case
Address loc;
loc.m_strIP = b->second.m_strIP;
loc.m_iPort = b->second.m_iPort;
m_pOutput->m_vLocation[i].insert(loc);
m_pOutput->m_piLocID[i] = b->first;
}
else
{
// otherwise find if the user-sepcified location is available
map<int, BUCKET>::iterator p = m_mBucket.begin();
for (; p != m_mBucket.end(); ++ p)
{
if ((p->second.m_strIP == m_pOutput->m_vLocation[i].begin()->m_strIP) && (p->second.m_iPort == m_pOutput->m_vLocation[i].begin()->m_iPort))
break;
}
if (p == m_mBucket.end())
{
Address loc;
loc.m_strIP = b->second.m_strIP;
loc.m_iPort = b->second.m_iPort;
m_pOutput->m_vLocation[i].insert(loc);
m_pOutput->m_piLocID[i] = b->first;
}
else
{
Address loc;
loc.m_strIP = p->second.m_strIP;
loc.m_iPort = p->second.m_iPort;
m_pOutput->m_vLocation[i].insert(loc);
m_pOutput->m_piLocID[i] = p->first;
}
}
if (++ b == m_mBucket.end())
b = m_mBucket.begin();
}
}
else if (m_iOutputType < 0)
{
char* localname = new char[m_pOutput->m_strPath.length() + m_pOutput->m_strName.length() + 64];
sprintf(localname, "%s/%s", m_pOutput->m_strPath.c_str(), m_pOutput->m_strName.c_str());
m_pOutputLoc = new char[strlen(localname) + 1];
memcpy(m_pOutputLoc, localname, strlen(localname) + 1);
}
return m_pOutput->m_iFileNum;
}
int DCClient::run(const SphereStream& input, SphereStream& output, const string& op, const int& rows, const char* param, const int& size, const int& type)
{
CGuard::enterCS(m_RunLock);
CGuard::leaveCS(m_RunLock);
m_iProcType = type;
m_strOperator = op;
m_pcParam = new char[size];
memcpy(m_pcParam, param, size);
m_iParamSize = size;
m_pInput = (SphereStream*)&input;
m_pOutput = &output;
m_iRows = rows;
m_iOutputType = m_pOutput->m_iFileNum;
// when processing files, data will not be moved
if (rows == 0)
m_bDataMove = false;
m_mpDS.clear();
m_mBucket.clear();
m_mSPE.clear();
int result = prepareInput();
if (result < 0)
return result;
cout << "JOB " << m_pInput->m_iFileNum << " " << m_pInput->m_llSize << " " << m_pInput->m_llRecNum << endl;
SectorMsg msg;
msg.setType(202); // locate available SPE
msg.setKey(m_pClient->m_iKey);
msg.m_iDataLength = SectorMsg::m_iHdrSize;
Address serv;
m_pClient->m_Routing.getPrimaryMaster(serv);
if (m_pClient->m_GMP.rpc(serv.m_strIP.c_str(), serv.m_iPort, &msg, &msg) < 0)
return SectorError::E_CONNECTION;
if (msg.getType() < 0)
return *(int32_t*)msg.getData();
m_iSPENum = (msg.m_iDataLength - 4) / 72;
if (0 == m_iSPENum)
return SectorError::E_RESOURCE;
result = prepareSPE(msg.getData());
if (result < 0)
return result;
result = segmentData();
if (result <= 0)
return result;
if (m_iOutputType == -1)
m_pOutput->init(m_mpDS.size());
result = prepareOutput(msg.getData());
if (result < 0)
return result;
m_iProgress = 0;
m_iAvgRunTime = -1;
m_iTotalDS = m_mpDS.size();
m_iTotalSPE = m_mSPE.size();
m_iAvailRes = 0;
m_bBucketHealth = true;
cout << m_mSPE.size() << " spes found! " << m_mpDS.size() << " data seg total." << endl;
// starting...
#ifndef WIN32
pthread_t scheduler;
pthread_create(&scheduler, NULL, run, this);
pthread_detach(scheduler);
#else
DWORD ThreadID;
CreateThread(NULL, 0, run, this, NULL, &ThreadID);
#endif
m_bOpened = true;
return 0;
}
int Slave::sendResultToBuckets(const int& buckets, const SPEResult& result, const SPEDestination& dest)
{
map<int, set<int> > ResByLoc;
map<int, int> SizeByLoc;
for (int i = 0; i < dest.m_iLocNum; ++ i)
{
ResByLoc[i].clear();
SizeByLoc[i] = 0;
}
// summarize information for each bucket, ignore empty buckets
for (int r = 0; r < buckets; ++ r)
{
int i = dest.m_piLocID[r];
if (0 != result.m_vDataLen[r])
{
ResByLoc[i].insert(r);
SizeByLoc[i] += result.m_vDataLen[r] + (result.m_vIndexLen[r] - 1) * 8;
}
}
unsigned int tn = 0;
map<int, set<int> >::iterator p = ResByLoc.begin();
while(!ResByLoc.empty())
{
int i = p->first;
if (++ p == ResByLoc.end())
p = ResByLoc.begin();
if (ResByLoc[i].empty())
{
//skip empty buckets
ResByLoc.erase(i);
SizeByLoc.erase(i);
continue;
}
// retrieve bucket location/address
char* dstip = dest.m_pcOutputLoc + i * 80;
int32_t dstport = *(int32_t*)(dest.m_pcOutputLoc + i * 80 + 68);
int32_t shufflerport = *(int32_t*)(dest.m_pcOutputLoc + i * 80 + 72);
int32_t session = *(int32_t*)(dest.m_pcOutputLoc + i * 80 + 76);
SectorMsg msg;
int32_t srcport = m_DataChn.getPort();
msg.setData(0, (char*)&srcport, 4);
msg.setData(4, (char*)&session, 4);
int totalnum = ResByLoc[i].size();
msg.setData(8, (char*)&totalnum, 4);
int totalsize = SizeByLoc[i];
msg.setData(12, (char*)&totalsize, 4);
msg.m_iDataLength = SectorMsg::m_iHdrSize + 16;
// request to send results to the slave node
msg.setType(1);
if (m_GMP.rpc(dstip, shufflerport, &msg, &msg) < 0)
return -1;
if (msg.getType() < 0)
{
// if all shufflers are busy, wait here a little while
if (++ tn >= ResByLoc.size())
{
tn = 0;
#ifndef WIN32
usleep(100000);
#else
Sleep(100);
#endif
}
continue;
}
if (!m_DataChn.isConnected(dstip, dstport))
{
if (m_DataChn.connect(dstip, dstport) < 0)
return -1;
}
// send results for one bucket a time
for (set<int>::iterator r = ResByLoc[i].begin(); r != ResByLoc[i].end(); ++ r)
{
int32_t id = *r;
m_DataChn.send(dstip, dstport, session, (char*)&id, 4);
m_DataChn.send(dstip, dstport, session, result.m_vData[id], result.m_vDataLen[id]);
m_DataChn.send(dstip, dstport, session, (char*)(result.m_vIndex[id] + 1), (result.m_vIndexLen[id] - 1) * 8);
}
// update total sent data
m_SlaveStat.updateIO(dstip, SizeByLoc[i], +SlaveStat::SYS_OUT);
ResByLoc.erase(i);
SizeByLoc.erase(i);
}
return 1;
}
DWORD WINAPI Slave::SPEShuffler(LPVOID p)
#endif
{
Slave* self = ((Param5*)p)->serv_instance;
int transid = ((Param5*)p)->transid;
string client_ip = ((Param5*)p)->client_ip;
int client_port = ((Param5*)p)->client_ctrl_port;
int client_data_port = ((Param5*)p)->client_data_port;
string path = ((Param5*)p)->path;
string localfile = ((Param5*)p)->filename;
int bucketnum = ((Param5*)p)->bucketnum;
CGMP* gmp = ((Param5*)p)->gmp;
string function = ((Param5*)p)->function;
int bucketid = ((Param5*)p)->bucketid;
const int key = ((Param5*)p)->key;
const int type = ((Param5*)p)->type;
string master_ip = ((Param5*)p)->master_ip;
int master_port = ((Param5*)p)->master_port;
queue<Bucket>* bq = NULL;
CMutex* bqlock = NULL;
CCond* bqcond = NULL;
int64_t* pendingSize = NULL;
pthread_t shufflerex;
bool init_success = true;
//set up data connection, for keep-alive purpose
if (self->m_DataChn.connect(client_ip, client_data_port) < 0)
{
init_success = false;
}
else
{
// read library files for MapReduce, no need for Sphere UDF
if (type == 1)
self->acceptLibrary(key, client_ip, client_data_port, transid);
bq = new queue<Bucket>;
bqlock = new CMutex;
bqcond = new CCond;
pendingSize = new int64_t;
*pendingSize = 0;
((Param5*)p)->bq = bq;
((Param5*)p)->bqlock = bqlock;
((Param5*)p)->bqcond = bqcond;
((Param5*)p)->pending = pendingSize;
#ifndef WIN32
pthread_create(&shufflerex, NULL, SPEShufflerEx, p);
#else
DWORD ThreadID;
shufflerex = CreateThread(NULL, 0, SPEShufflerEx, p, NULL, &ThreadID);
#endif
self->m_SectorLog << LogStart(LogLevel::SCREEN) << "SPE Shuffler " << path << " " << localfile << " " << bucketnum << LogEnd();
}
while (init_success)
{
string speip;
int speport;
SectorMsg msg;
int msgid;
int r = gmp->recvfrom(speip, speport, msgid, &msg, false);
// client releases the task or client has already been shutdown
if (((r > 0) && (speip == client_ip) && (speport == client_port))
|| ((r < 0) && (!self->m_DataChn.isConnected(client_ip, client_data_port))))
{
Bucket b;
b.totalnum = -1;
b.totalsize = 0;
bqlock->acquire();
bq->push(b);
bqcond->signal();
bqlock->release();
break;
}
if (r < 0)
continue;
if (*pendingSize > 256000000)
{
// too many incoming results, ask the sender to wait
// the receiver buffer size threshold is set to 256MB. This prevents the shuffler from being overflowed
// it also helps direct the traffic to less congested shuffler and leads to better load balance
msg.setType(-msg.getType());
gmp->sendto(speip, speport, msgid, &msg);
}
else
{
Bucket b;
b.totalnum = *(int32_t*)(msg.getData() + 8);;
b.totalsize = *(int32_t*)(msg.getData() + 12);
b.src_ip = speip;
b.src_dataport = *(int32_t*)msg.getData();
b.session = *(int32_t*)(msg.getData() + 4);
gmp->sendto(speip, speport, msgid, &msg);
if (!self->m_DataChn.isConnected(speip, b.src_dataport))
self->m_DataChn.connect(speip, b.src_dataport);
bqlock->acquire();
bq->push(b);
*pendingSize += b.totalsize;
bqcond->signal();
bqlock->release();
}
}
if (init_success)
{
#ifndef WIN32
pthread_join(shufflerex, NULL);
#else
WaitForSingleObject(shufflerex, INFINITE);
#endif
delete bqlock;
delete bqcond;
delete pendingSize;
SectorMsg msg;
msg.setType(1); // success, return result
msg.setData(0, (char*)&(bucketid), 4);
int progress = 100;
msg.setData(4, (char*)&progress, 4);
msg.m_iDataLength = SectorMsg::m_iHdrSize + 8;
int id = 0;
self->m_GMP.sendto(client_ip.c_str(), client_port, id, &msg);
self->m_SectorLog << LogStart(LogLevel::LEVEL_3) << "bucket completed 100 " << client_ip << " " << client_port << LogEnd();
}
gmp->close();
delete gmp;
self->reportSphere(master_ip, master_port, transid);
// clear this transaction
self->m_TransManager.updateSlave(transid, self->m_iSlaveID);
return NULL;
}
int Slave::readSectorFile(const string& filename, const int64_t& offset, const int64_t& size, char* buf)
{
SectorMsg msg;
msg.setType(110); // open the index file
msg.setKey(0);
int32_t mode = 1;
msg.setData(0, (char*)&mode, 4);
int32_t port = m_DataChn.getPort();
msg.setData(4, (char*)&port, 4);
int32_t len_name = filename.length() + 1;
msg.setData(8, (char*)&len_name, 4);
msg.setData(12, filename.c_str(), len_name);
int32_t len_opt = 0;
msg.setData(12 + len_name, (char*)&len_opt, 4);
Address addr;
m_Routing.lookup(filename, addr);
if (m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0)
return -1;
if (msg.getType() < 0)
return -1;
int32_t session = *(int32_t*)msg.getData();
string srcip = msg.getData() + 24;
int32_t srcport = *(int32_t*)(msg.getData() + 64 + 24);
// connect to the slave node with the file.
if (!m_DataChn.isConnected(srcip, srcport))
{
if (m_DataChn.connect(srcip, srcport) < 0)
return -1;
}
int32_t cmd = 1;
m_DataChn.send(srcip, srcport, session, (char*)&cmd, 4);
char req[16];
*(int64_t*)req = offset;
*(int64_t*)(req + 8) = size;
if (m_DataChn.send(srcip, srcport, session, req, 16) < 0)
return -1;
int response = -1;
if (m_DataChn.recv4(srcip, srcport, session, response) < 0)
return -1;
char* tmp = NULL;
int recvsize = size;
if (m_DataChn.recv(srcip, srcport, session, tmp, recvsize) < 0)
return -1;
if (recvsize == size)
memcpy(buf, tmp, size);
delete [] tmp;
// file close command: 5
cmd = 5;
m_DataChn.send(srcip, srcport, session, (char*)&cmd, 4);
m_DataChn.recv4(srcip, srcport, session, response);
// update total received data
m_SlaveStat.updateIO(srcip, size, +SlaveStat::SYS_IN);
if (recvsize != size)
return -1;
return size;
}
int Slave::report(const string& master_ip, const int& master_port, const int32_t& transid, const vector<string>& filelist, const int32_t& change)
{
vector<string> serlist;
if (change != FileChangeType::FILE_UPDATE_NO)
{
for (vector<string>::const_iterator i = filelist.begin(); i != filelist.end(); ++ i)
{
struct stat s;
if (-1 == stat ((m_strHomeDir + *i).c_str(), &s))
continue;
SNode sn;
sn.m_strName = *i;
sn.m_bIsDir = S_ISDIR(s.st_mode) ? 1 : 0;
sn.m_llTimeStamp = s.st_mtime;
sn.m_llSize = s.st_size;
Address addr;
addr.m_strIP = "127.0.0.1";
addr.m_iPort = 0;
sn.m_sLocation.insert(addr);
if (change == FileChangeType::FILE_UPDATE_WRITE)
{
// file may be created on write; in this case, create a new meta entry instead of update non-existing one
if (m_pLocalFile->update(sn.m_strName, sn.m_llTimeStamp, sn.m_llSize) < 0)
m_pLocalFile->create(sn);
}
else if (change == FileChangeType::FILE_UPDATE_NEW)
m_pLocalFile->create(sn);
else if (change == FileChangeType::FILE_UPDATE_REPLICA)
m_pLocalFile->create(sn);
char* buf = NULL;
sn.serialize(buf);
serlist.push_back(buf);
delete [] buf;
}
}
SectorMsg msg;
msg.setType(1);
msg.setKey(0);
msg.setData(0, (char*)&transid, 4);
msg.setData(4, (char*)&m_iSlaveID, 4);
msg.setData(8, (char*)&change, 4);
int32_t num = serlist.size();
msg.setData(12, (char*)&num, 4);
int pos = 16;
for (vector<string>::iterator i = serlist.begin(); i != serlist.end(); ++ i)
{
int32_t bufsize = i->length() + 1;
msg.setData(pos, (char*)&bufsize, 4);
msg.setData(pos + 4, i->c_str(), bufsize);
pos += bufsize + 4;
}
//TODO: if the current master is down, try a different master
if (m_GMP.rpc(master_ip.c_str(), master_port, &msg, &msg) < 0)
return -1;
if (msg.getType() < 0)
return *(int32_t*)msg.getData();
return 1;
}
void* Slave::copy(void* p)
{
Slave* self = ((Param3*)p)->serv_instance;
int transid = ((Param3*)p)->transid;
string src = ((Param3*)p)->src;
string dst = ((Param3*)p)->dst;
string master_ip = ((Param3*)p)->master_ip;
int master_port = ((Param3*)p)->master_port;
delete (Param3*)p;
if (src.c_str()[0] == '\0')
src = "/" + src;
if (dst.c_str()[0] == '\0')
dst = "/" + dst;
SNode tmp;
if (self->m_pLocalFile->lookup(src.c_str(), tmp) >= 0)
{
//if file is local, copy directly
//note that in this case, src != dst, therefore this is a regular "cp" command, not a system replication
//TODO: check disk space
self->createDir(dst.substr(0, dst.rfind('/')));
string rhome = self->reviseSysCmdPath(self->m_strHomeDir);
string rsrc = self->reviseSysCmdPath(src);
string rdst = self->reviseSysCmdPath(dst);
system(("cp " + rhome + rsrc + " " + rhome + rdst).c_str());
// if the file has been modified during the replication, remove this replica
int type = (src == dst) ? +FileChangeType::FILE_UPDATE_REPLICA : +FileChangeType::FILE_UPDATE_NEW;
struct stat64 s;
if (stat64((self->m_strHomeDir + dst).c_str(), &s) < 0)
type = +FileChangeType::FILE_UPDATE_NO;
if (self->report(master_ip, master_port, transid, dst, type) < 0)
system(("rm " + rhome + rdst).c_str());
// clear this transaction
self->m_TransManager.updateSlave(transid, self->m_iSlaveID);
return NULL;
}
bool success = true;
queue<string> tr;
tr.push(src);
while (!tr.empty())
{
string src_path = tr.front();
tr.pop();
// try list this path
SectorMsg msg;
msg.setType(101);
msg.setKey(0);
msg.setData(0, src_path.c_str(), src_path.length() + 1);
Address addr;
self->m_Routing.lookup(src_path, addr);
if (self->m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0)
{
success = false;
break;
}
if (msg.getType() >= 0)
{
// if this is a directory, put all files and sub-drectories into the queue of files to be copied
string filelist = msg.getData();
unsigned int s = 0;
while (s < filelist.length())
{
int t = filelist.find(';', s);
SNode sn;
sn.deserialize(filelist.substr(s, t - s).c_str());
tr.push(src_path + "/" + sn.m_strName);
s = t + 1;
}
continue;
}
// open the file and copy it to local
msg.setType(110);
msg.setKey(0);
int32_t mode = SF_MODE::READ;
msg.setData(0, (char*)&mode, 4);
int64_t reserve = 0;
msg.setData(4, (char*)&reserve, 8);
int32_t localport = self->m_DataChn.getPort();
msg.setData(12, (char*)&localport, 4);
msg.setData(16, "\0", 1);
msg.setData(80, src_path.c_str(), src_path.length() + 1);
if ((self->m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0) || (msg.getType() < 0))
{
success = false;
break;
}
int32_t session = *(int32_t*)msg.getData();
int64_t size = *(int64_t*)(msg.getData() + 4);
time_t ts = *(int64_t*)(msg.getData() + 12);
string ip = msg.getData() + 24;
int32_t port = *(int32_t*)(msg.getData() + 64 + 24);
if (!self->m_DataChn.isConnected(ip, port))
{
if (self->m_DataChn.connect(ip, port) < 0)
{
success = false;
break;
}
}
// download command: 3
int32_t cmd = 3;
self->m_DataChn.send(ip, port, session, (char*)&cmd, 4);
int64_t offset = 0;
self->m_DataChn.send(ip, port, session, (char*)&offset, 8);
int response = -1;
if ((self->m_DataChn.recv4(ip, port, session, response) < 0) || (-1 == response))
{
success = false;
break;
}
string dst_path = dst;
if (src != src_path)
dst_path += "/" + src_path.substr(src.length() + 1, src_path.length() - src.length() - 1);
//copy to .tmp first, then move to real location
self->createDir(string(".tmp") + dst_path.substr(0, dst_path.rfind('/')));
fstream ofs;
ofs.open((self->m_strHomeDir + ".tmp" + dst_path).c_str(), ios::out | ios::binary | ios::trunc);
int64_t unit = 64000000; //send 64MB each time
int64_t torecv = size;
int64_t recd = 0;
while (torecv > 0)
{
int64_t block = (torecv < unit) ? torecv : unit;
if (self->m_DataChn.recvfile(ip, port, session, ofs, offset + recd, block) < 0)
{
success = false;
break;
}
recd += block;
torecv -= block;
}
ofs.close();
// update total received data size
self->m_SlaveStat.updateIO(ip, size, +SlaveStat::SYS_IN);
cmd = 5;
self->m_DataChn.send(ip, port, session, (char*)&cmd, 4);
self->m_DataChn.recv4(ip, port, session, cmd);
if (src == dst)
{
//utime: update timestamp according to the original copy, for replica only; files created by "cp" have new timestamp
utimbuf ut;
ut.actime = ts;
ut.modtime = ts;
utime((self->m_strHomeDir + ".tmp" + dst_path).c_str(), &ut);
}
}
string rhome = self->reviseSysCmdPath(self->m_strHomeDir);
string rfile = self->reviseSysCmdPath(dst);
if (success)
{
// move from temporary dir to the real dir when the copy is completed
self->createDir(dst.substr(0, dst.rfind('/')));
system(("mv " + rhome + ".tmp" + rfile + " " + rhome + rfile).c_str());
// if the file has been modified during the replication, remove this replica
int32_t type = (src == dst) ? +FileChangeType::FILE_UPDATE_REPLICA : +FileChangeType::FILE_UPDATE_NEW;
if (self->report(master_ip, master_port, transid, dst, type) < 0)
unlink((rhome + rfile).c_str());
}
else
{
// failed, remove all temporary files
system(("rm -rf " + rhome + ".tmp" + rfile).c_str());
self->report(master_ip, master_port, transid, "", +FileChangeType::FILE_UPDATE_NO);
}
// clear this transaction
self->m_TransManager.updateSlave(transid, self->m_iSlaveID);
return NULL;
}
int Slave::SPEReadData(const string& datafile, const int64_t& offset, int& size, int64_t* index, const int64_t& totalrows, char*& block)
{
SNode sn;
string idxfile = datafile + ".idx";
//read index
if (m_pLocalFile->lookup(idxfile.c_str(), sn) >= 0)
{
fstream idx;
idx.open((m_strHomeDir + idxfile).c_str(), ios::in | ios::binary);
if (idx.bad() || idx.fail())
return -1;
idx.seekg(offset * 8);
idx.read((char*)index, (totalrows + 1) * 8);
idx.close();
}
else if (size) {
SectorMsg msg;
msg.setType(110); // open the index file
msg.setKey(0);
int32_t mode = 1;
msg.setData(0, (char*)&mode, 4);
int32_t port = m_DataChn.getPort();
msg.setData(4, (char*)&port, 4);
msg.setData(8, "\0", 1);
msg.setData(72, idxfile.c_str(), idxfile.length() + 1);
Address addr;
m_Routing.lookup(idxfile, addr);
if (m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0)
return -1;
if (msg.getType() < 0)
return -1;
string srcip = msg.getData();
int srcport = *(int*)(msg.getData() + 64);
int session = *(int*)(msg.getData() + 68);
// cout << "rendezvous connect " << srcip << " " << srcport << endl;
if (m_DataChn.connect(srcip, srcport) < 0)
return -1;
int32_t cmd = 1;
m_DataChn.send(srcip, srcport, session, (char*)&cmd, 4);
int response = -1;
if (m_DataChn.recv4(srcip, srcport, session, response) < 0)
return -1;
char req[16];
*(int64_t*)req = offset * 8;
*(int64_t*)(req + 8) = (totalrows + 1) * 8;
if (m_DataChn.send(srcip, srcport, session, req, 16) < 0)
return -1;
char* tmp = NULL;
int size = (totalrows + 1) * 8;
if (m_DataChn.recv(srcip, srcport, session, tmp, size) < 0)
return -1;
if (size > 0)
memcpy((char*)index, tmp, size);
delete [] tmp;
// file close command: 5
cmd = 5;
m_DataChn.send(srcip, srcport, session, (char*)&cmd, 4);
m_DataChn.recv4(srcip, srcport, session, response);
// update total received data
m_SlaveStat.updateIO(srcip, (totalrows + 1) * 8, 0);
} else {
// no index, so rows are bytes
size = totalrows;
index[0] = 0;
goto Lreaddata;
}
size = index[totalrows] - index[0];
Lreaddata:
block = new char[size];
// read data file
if (m_pLocalFile->lookup(datafile.c_str(), sn) >= 0)
{
fstream ifs;
ifs.open((m_strHomeDir + datafile).c_str(), ios::in | ios::binary);
if (ifs.bad() || ifs.fail())
return -1;
ifs.seekg(index[0]);
ifs.read(block, size);
ifs.close();
}
else
{
SectorMsg msg;
msg.setType(110); // open the index file
msg.setKey(0);
int32_t mode = 1;
msg.setData(0, (char*)&mode, 4);
int32_t port = m_DataChn.getPort();
msg.setData(4, (char*)&port, 4);
msg.setData(8, "\0", 1);
msg.setData(72, datafile.c_str(), datafile.length() + 1);
Address addr;
m_Routing.lookup(datafile, addr);
if (m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0)
return -1;
if (msg.getType() < 0)
return -1;
string srcip = msg.getData();
int srcport = *(int*)(msg.getData() + 64);
int session = *(int*)(msg.getData() + 68);
// cout << "rendezvous connect " << srcip << " " << srcport << endl;
if (m_DataChn.connect(srcip, srcport) < 0)
return -1;
int32_t cmd = 1;
m_DataChn.send(srcip, srcport, session, (char*)&cmd, 4);
int response = -1;
if (m_DataChn.recv4(srcip, srcport, session, response) < 0)
return -1;
char req[16];
*(int64_t*)req = index[0];
*(int64_t*)(req + 8) = index[totalrows] - index[0];
if (m_DataChn.send(srcip, srcport, session, req, 16) < 0)
return -1;
char* tmp = NULL;
int size = index[totalrows] - index[0];
if (m_DataChn.recv(srcip, srcport, session, tmp, size) < 0)
return -1;
if (size > 0)
memcpy(block, tmp, size);
delete [] tmp;
// file close command: 5
cmd = 5;
m_DataChn.send(srcip, srcport, session, (char*)&cmd, 4);
m_DataChn.recv4(srcip, srcport, session, response);
// update total received data
m_SlaveStat.updateIO(srcip, index[totalrows] - index[0], 0);
}
return totalrows;
}
void* Slave::SPEShuffler(void* p)
{
Slave* self = ((Param5*)p)->serv_instance;
string client_ip = ((Param5*)p)->client_ip;
int client_port = ((Param5*)p)->client_ctrl_port;
int client_data_port = ((Param5*)p)->client_data_port;
string path = ((Param5*)p)->path;
string localfile = ((Param5*)p)->filename;
// int bucketnum = ((Param5*)p)->bucketnum;
CGMP* gmp = ((Param5*)p)->gmp;
string function = ((Param5*)p)->function;
//set up data connection, for keep-alive purpose
if (self->m_DataChn.connect(client_ip, client_data_port) < 0)
return NULL;
queue<Bucket>* bq = new queue<Bucket>;
pthread_mutex_t* bqlock = new pthread_mutex_t;
pthread_mutex_init(bqlock, NULL);
pthread_cond_t* bqcond = new pthread_cond_t;
pthread_cond_init(bqcond, NULL);
int64_t* pendingSize = new int64_t;
*pendingSize = 0;
((Param5*)p)->bq = bq;
((Param5*)p)->bqlock = bqlock;
((Param5*)p)->bqcond = bqcond;
((Param5*)p)->pending = pendingSize;
pthread_t ex;
pthread_create(&ex, NULL, SPEShufflerEx, p);
pthread_detach(ex);
// cout << "SPE Shuffler " << path << " " << localfile << " " << bucketnum << endl;
while (true)
{
string speip;
int speport;
SectorMsg msg;
int msgid;
int r = gmp->recvfrom(speip, speport, msgid, &msg, false);
// client releases the task or client has already been shutdown
if (((r > 0) && (speip == client_ip) && (speport == client_port))
|| ((r < 0) && (!self->m_DataChn.isConnected(client_ip, client_data_port))))
{
Bucket b;
b.totalnum = -1;
b.totalsize = 0;
pthread_mutex_lock(bqlock);
bq->push(b);
pthread_cond_signal(bqcond);
pthread_mutex_unlock(bqlock);
break;
}
if (r < 0)
continue;
if (*pendingSize > 256000000)
{
// too many incoming results, ask the sender to wait
// the receiver buffer size threshold is set to 256MB. This prevents the shuffler from being overflowed
// it also helps direct the traffic to less congested shuffler and leads to better load balance
msg.setType(-msg.getType());
gmp->sendto(speip, speport, msgid, &msg);
}
else
{
Bucket b;
b.totalnum = *(int32_t*)(msg.getData() + 8);;
b.totalsize = *(int32_t*)(msg.getData() + 12);
b.src_ip = speip;
b.src_dataport = *(int32_t*)msg.getData();
b.session = *(int32_t*)(msg.getData() + 4);
gmp->sendto(speip, speport, msgid, &msg);
if (!self->m_DataChn.isConnected(speip, b.src_dataport))
self->m_DataChn.connect(speip, b.src_dataport);
pthread_mutex_lock(bqlock);
bq->push(b);
*pendingSize += b.totalsize;
pthread_cond_signal(bqcond);
pthread_mutex_unlock(bqlock);
}
}
gmp->close();
delete gmp;
return NULL;
}
DWORD WINAPI Slave::copy(LPVOID p)
#endif
{
Slave* self = ((Param3*)p)->serv_instance;
int transid = ((Param3*)p)->transid;
int dir = ((Param3*)p)->dir;
string src = ((Param3*)p)->src;
string dst = ((Param3*)p)->dst;
string master_ip = ((Param3*)p)->master_ip;
int master_port = ((Param3*)p)->master_port;
delete (Param3*)p;
if (src.c_str()[0] == '\0')
src = "/" + src;
if (dst.c_str()[0] == '\0')
dst = "/" + dst;
bool success = true;
queue<string> tr; // files to be replicated
queue<string> td; // directories to be explored
if (dir > 0)
td.push(src);
else
tr.push(src);
while (!td.empty())
{
// If the file to be replicated is a directory, recursively list all files first
string src_path = td.front();
td.pop();
// try list this path
SectorMsg msg;
msg.setType(101);
msg.setKey(0);
msg.setData(0, src_path.c_str(), src_path.length() + 1);
Address addr;
self->m_Routing.lookup(src_path, addr);
if (self->m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0)
{
success = false;
break;
}
// the master only returns positive if this is a directory
if (msg.getType() >= 0)
{
// if this is a directory, create it, and put all files and sub-directories into the queue of files to be copied
// create a local dir
string dst_path = dst;
if (src != src_path)
dst_path += "/" + src_path.substr(src.length() + 1, src_path.length() - src.length() - 1);
//create at .tmp first, then move to real location
self->createDir(string(".tmp") + dst_path);
string filelist = msg.getData();
unsigned int s = 0;
while (s < filelist.length())
{
int t = filelist.find(';', s);
SNode sn;
sn.deserialize(filelist.substr(s, t - s).c_str());
if (sn.m_bIsDir)
td.push(src_path + "/" + sn.m_strName);
else
tr.push(src_path + "/" + sn.m_strName);
s = t + 1;
}
continue;
}
}
while (!tr.empty())
{
string src_path = tr.front();
tr.pop();
SNode tmp;
if (self->m_pLocalFile->lookup(src_path.c_str(), tmp) >= 0)
{
//if file is local, copy directly
//note that in this case, src != dst, therefore this is a regular "cp" command, not a system replication
//IMPORTANT!!!
//local files must be read directly from local disk, and cannot be read via datachn due to its limitation
string dst_path = dst;
if (src != src_path)
dst_path += "/" + src_path.substr(src.length() + 1, src_path.length() - src.length() - 1);
//copy to .tmp first, then move to real location
self->createDir(string(".tmp") + dst_path.substr(0, dst_path.rfind('/')));
LocalFS::copy(self->m_strHomeDir + src_path, self->m_strHomeDir + ".tmp" + dst_path);
}
else
{
// open the file and copy it to local
SectorMsg msg;
msg.setType(110);
msg.setKey(0);
int32_t mode = SF_MODE::READ;
msg.setData(0, (char*)&mode, 4);
int32_t localport = self->m_DataChn.getPort();
msg.setData(4, (char*)&localport, 4);
int32_t len_name = src_path.length() + 1;
msg.setData(8, (char*)&len_name, 4);
msg.setData(12, src_path.c_str(), len_name);
int32_t len_opt = 0;
msg.setData(12 + len_name, (char*)&len_opt, 4);
Address addr;
self->m_Routing.lookup(src_path, addr);
if ((self->m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0) || (msg.getType() < 0))
{
success = false;
break;
}
int32_t session = *(int32_t*)msg.getData();
int64_t size = *(int64_t*)(msg.getData() + 4);
time_t ts = *(int64_t*)(msg.getData() + 12);
string ip = msg.getData() + 24;
int32_t port = *(int32_t*)(msg.getData() + 64 + 24);
if (!self->m_DataChn.isConnected(ip, port))
{
if (self->m_DataChn.connect(ip, port) < 0)
{
success = false;
break;
}
}
// download command: 3
int32_t cmd = 3;
self->m_DataChn.send(ip, port, session, (char*)&cmd, 4);
int64_t offset = 0;
self->m_DataChn.send(ip, port, session, (char*)&offset, 8);
int response = -1;
if ((self->m_DataChn.recv4(ip, port, session, response) < 0) || (-1 == response))
{
success = false;
break;
}
string dst_path = dst;
if (src != src_path)
dst_path += "/" + src_path.substr(src.length() + 1, src_path.length() - src.length() - 1);
//copy to .tmp first, then move to real location
self->createDir(string(".tmp") + dst_path.substr(0, dst_path.rfind('/')));
fstream ofs;
ofs.open((self->m_strHomeDir + ".tmp" + dst_path).c_str(), ios::out | ios::binary | ios::trunc);
int64_t unit = 64000000; //send 64MB each time
int64_t torecv = size;
int64_t recd = 0;
while (torecv > 0)
{
int64_t block = (torecv < unit) ? torecv : unit;
if (self->m_DataChn.recvfile(ip, port, session, ofs, offset + recd, block) < 0)
{
success = false;
break;
}
recd += block;
torecv -= block;
}
ofs.close();
// update total received data size
self->m_SlaveStat.updateIO(ip, size, +SlaveStat::SYS_IN);
cmd = 5;
self->m_DataChn.send(ip, port, session, (char*)&cmd, 4);
self->m_DataChn.recv4(ip, port, session, cmd);
if (src == dst)
{
//utime: update timestamp according to the original copy, for replica only; files created by "cp" have new timestamp
utimbuf ut;
ut.actime = ts;
ut.modtime = ts;
utime((self->m_strHomeDir + ".tmp" + dst_path).c_str(), &ut);
}
}
}
if (success)
{
// move from temporary dir to the real dir when the copy is completed
self->createDir(dst.substr(0, dst.rfind('/')));
LocalFS::rename(self->m_strHomeDir + ".tmp" + dst, self->m_strHomeDir + dst);
// if the file has been modified during the replication, remove this replica
int32_t type = (src == dst) ? +FileChangeType::FILE_UPDATE_REPLICA : +FileChangeType::FILE_UPDATE_NEW;
if (self->report(master_ip, master_port, transid, dst, type) < 0)
LocalFS::erase(self->m_strHomeDir + dst);
}
else
{
// failed, remove all temporary files
LocalFS::erase(self->m_strHomeDir + ".tmp" + dst);
self->report(master_ip, master_port, transid, "", +FileChangeType::FILE_UPDATE_NO);
}
// clear this transaction
self->m_TransManager.updateSlave(transid, self->m_iSlaveID);
return NULL;
}
void Slave::run()
{
string ip;
int port;
int32_t id;
SectorMsg* msg = new SectorMsg;
msg->resize(65536);
cout << "slave process: " << "GMP " << m_iLocalPort << " DATA " << m_DataChn.getPort() << endl;
m_bRunning = true;
pthread_t worker_thread;
#ifndef WIN32 // <slr>
pthread_create(&worker_thread, NULL, worker, this);
#else
unsigned int ThreadID;
worker_thread = (HANDLE)_beginthreadex(NULL, 0, worker, this, NULL, &ThreadID);
#endif
while (m_bRunning)
{
if (m_GMP.recvfrom(ip, port, id, msg) < 0)
break;
m_SectorLog << LogStringTag(LogTag::START, LogLevel::SCREEN) << "recv cmd " << ip << " " << port << " type " << msg->getType() << LogStringTag(LogTag::END);
// a slave only accepts commands from the masters
Address addr;
addr.m_strIP = ip;
addr.m_iPort = port;
if (m_Routing.getRouterID(addr) < 0)
continue;
switch (msg->getType() / 100)
{
case 0:
processSysCmd(ip, port, id, msg);
break;
case 1:
processFSCmd(ip, port, id, msg);
break;
case 2:
processDCCmd(ip, port, id, msg);
break;
case 3:
processDBCmd(ip, port, id, msg);
break;
case 10:
processMCmd(ip, port, id, msg);
break;
#ifdef DEBUG
processDebugCmd(ip, port, id, msg);
break;
#endif
default:
break;
}
}
delete msg;
#ifndef WIN32
pthread_join(worker_thread, NULL);
#else
WaitForSingleObject(worker_thread, INFINITE);
#endif
// TODO: check and cancel all file&spe threads
cout << "slave is stopped by master\n";
}