int Client::list(const string& path, vector<SNode>& attr)
{
string revised_path = Metadata::revisePath(path);
SectorMsg msg;
msg.resize(65536);
msg.setType(101);
msg.setKey(m_iKey);
msg.setData(0, revised_path.c_str(), revised_path.length() + 1);
Address serv;
m_Routing.lookup(revised_path, serv);
login(serv.m_strIP, serv.m_iPort);
if (m_GMP.rpc(serv.m_strIP.c_str(), serv.m_iPort, &msg, &msg) < 0)
return SectorError::E_CONNECTION;
if (msg.getType() < 0)
return *(int32_t*)(msg.getData());
string filelist = msg.getData();
unsigned int s = 0;
while (s < filelist.length())
{
int t = filelist.find(';', s);
SNode sn;
sn.deserialize(filelist.substr(s, t - s).c_str());
attr.insert(attr.end(), sn);
s = t + 1;
}
return attr.size();
}
int Client::stat(const string& path, SNode& attr)
{
string revised_path = Metadata::revisePath(path);
SectorMsg msg;
msg.resize(65536);
msg.setType(102);
msg.setKey(m_iKey);
msg.setData(0, revised_path.c_str(), revised_path.length() + 1);
Address serv;
m_Routing.lookup(revised_path, serv);
login(serv.m_strIP, serv.m_iPort);
if (m_GMP.rpc(serv.m_strIP.c_str(), serv.m_iPort, &msg, &msg) < 0)
return SectorError::E_CONNECTION;
if (msg.getType() < 0)
return *(int32_t*)(msg.getData());
attr.deserialize(msg.getData());
int n = (msg.m_iDataLength - SectorMsg::m_iHdrSize - 128) / 68;
char* al = msg.getData() + 128;
for (int i = 0; i < n; ++ i)
{
Address addr;
addr.m_strIP = al + 68 * i;
addr.m_iPort = *(int32_t*)(al + 68 * i + 64);
attr.m_sLocation.insert(addr);
}
// check local cache: updated files may not be sent to the master yet
m_StatCache.stat(path, attr);
return 0;
}
DWORD WINAPI Slave::copy(LPVOID p)
#endif
{
Slave* self = ((Param3*)p)->serv_instance;
int transid = ((Param3*)p)->transid;
int dir = ((Param3*)p)->dir;
string src = ((Param3*)p)->src;
string dst = ((Param3*)p)->dst;
string master_ip = ((Param3*)p)->master_ip;
int master_port = ((Param3*)p)->master_port;
delete (Param3*)p;
if (src.c_str()[0] == '\0')
src = "/" + src;
if (dst.c_str()[0] == '\0')
dst = "/" + dst;
bool success = true;
queue<string> tr; // files to be replicated
queue<string> td; // directories to be explored
if (dir > 0)
td.push(src);
else
tr.push(src);
while (!td.empty())
{
// If the file to be replicated is a directory, recursively list all files first
string src_path = td.front();
td.pop();
// try list this path
SectorMsg msg;
msg.setType(101);
msg.setKey(0);
msg.setData(0, src_path.c_str(), src_path.length() + 1);
Address addr;
self->m_Routing.lookup(src_path, addr);
if (self->m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0)
{
success = false;
break;
}
// the master only returns positive if this is a directory
if (msg.getType() >= 0)
{
// if this is a directory, create it, and put all files and sub-directories into the queue of files to be copied
// create a local dir
string dst_path = dst;
if (src != src_path)
dst_path += "/" + src_path.substr(src.length() + 1, src_path.length() - src.length() - 1);
//create at .tmp first, then move to real location
self->createDir(string(".tmp") + dst_path);
string filelist = msg.getData();
unsigned int s = 0;
while (s < filelist.length())
{
int t = filelist.find(';', s);
SNode sn;
sn.deserialize(filelist.substr(s, t - s).c_str());
if (sn.m_bIsDir)
td.push(src_path + "/" + sn.m_strName);
else
tr.push(src_path + "/" + sn.m_strName);
s = t + 1;
}
continue;
}
}
while (!tr.empty())
{
string src_path = tr.front();
tr.pop();
SNode tmp;
if (self->m_pLocalFile->lookup(src_path.c_str(), tmp) >= 0)
{
//if file is local, copy directly
//note that in this case, src != dst, therefore this is a regular "cp" command, not a system replication
//IMPORTANT!!!
//local files must be read directly from local disk, and cannot be read via datachn due to its limitation
string dst_path = dst;
if (src != src_path)
dst_path += "/" + src_path.substr(src.length() + 1, src_path.length() - src.length() - 1);
//copy to .tmp first, then move to real location
self->createDir(string(".tmp") + dst_path.substr(0, dst_path.rfind('/')));
LocalFS::copy(self->m_strHomeDir + src_path, self->m_strHomeDir + ".tmp" + dst_path);
}
else
{
// open the file and copy it to local
SectorMsg msg;
msg.setType(110);
msg.setKey(0);
int32_t mode = SF_MODE::READ;
msg.setData(0, (char*)&mode, 4);
int32_t localport = self->m_DataChn.getPort();
msg.setData(4, (char*)&localport, 4);
int32_t len_name = src_path.length() + 1;
msg.setData(8, (char*)&len_name, 4);
msg.setData(12, src_path.c_str(), len_name);
int32_t len_opt = 0;
msg.setData(12 + len_name, (char*)&len_opt, 4);
Address addr;
self->m_Routing.lookup(src_path, addr);
if ((self->m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0) || (msg.getType() < 0))
{
success = false;
break;
}
int32_t session = *(int32_t*)msg.getData();
int64_t size = *(int64_t*)(msg.getData() + 4);
time_t ts = *(int64_t*)(msg.getData() + 12);
string ip = msg.getData() + 24;
int32_t port = *(int32_t*)(msg.getData() + 64 + 24);
if (!self->m_DataChn.isConnected(ip, port))
{
if (self->m_DataChn.connect(ip, port) < 0)
{
success = false;
break;
}
}
// download command: 3
int32_t cmd = 3;
self->m_DataChn.send(ip, port, session, (char*)&cmd, 4);
int64_t offset = 0;
self->m_DataChn.send(ip, port, session, (char*)&offset, 8);
int response = -1;
if ((self->m_DataChn.recv4(ip, port, session, response) < 0) || (-1 == response))
{
success = false;
break;
}
string dst_path = dst;
if (src != src_path)
dst_path += "/" + src_path.substr(src.length() + 1, src_path.length() - src.length() - 1);
//copy to .tmp first, then move to real location
self->createDir(string(".tmp") + dst_path.substr(0, dst_path.rfind('/')));
fstream ofs;
ofs.open((self->m_strHomeDir + ".tmp" + dst_path).c_str(), ios::out | ios::binary | ios::trunc);
int64_t unit = 64000000; //send 64MB each time
int64_t torecv = size;
int64_t recd = 0;
while (torecv > 0)
{
int64_t block = (torecv < unit) ? torecv : unit;
if (self->m_DataChn.recvfile(ip, port, session, ofs, offset + recd, block) < 0)
{
success = false;
break;
}
recd += block;
torecv -= block;
}
ofs.close();
// update total received data size
self->m_SlaveStat.updateIO(ip, size, +SlaveStat::SYS_IN);
cmd = 5;
self->m_DataChn.send(ip, port, session, (char*)&cmd, 4);
self->m_DataChn.recv4(ip, port, session, cmd);
if (src == dst)
{
//utime: update timestamp according to the original copy, for replica only; files created by "cp" have new timestamp
utimbuf ut;
ut.actime = ts;
ut.modtime = ts;
utime((self->m_strHomeDir + ".tmp" + dst_path).c_str(), &ut);
}
}
}
if (success)
{
// move from temporary dir to the real dir when the copy is completed
self->createDir(dst.substr(0, dst.rfind('/')));
LocalFS::rename(self->m_strHomeDir + ".tmp" + dst, self->m_strHomeDir + dst);
// if the file has been modified during the replication, remove this replica
int32_t type = (src == dst) ? +FileChangeType::FILE_UPDATE_REPLICA : +FileChangeType::FILE_UPDATE_NEW;
if (self->report(master_ip, master_port, transid, dst, type) < 0)
LocalFS::erase(self->m_strHomeDir + dst);
}
else
{
// failed, remove all temporary files
LocalFS::erase(self->m_strHomeDir + ".tmp" + dst);
self->report(master_ip, master_port, transid, "", +FileChangeType::FILE_UPDATE_NO);
}
// clear this transaction
self->m_TransManager.updateSlave(transid, self->m_iSlaveID);
return NULL;
}
void* Slave::copy(void* p)
{
Slave* self = ((Param3*)p)->serv_instance;
int transid = ((Param3*)p)->transid;
string src = ((Param3*)p)->src;
string dst = ((Param3*)p)->dst;
string master_ip = ((Param3*)p)->master_ip;
int master_port = ((Param3*)p)->master_port;
delete (Param3*)p;
if (src.c_str()[0] == '\0')
src = "/" + src;
if (dst.c_str()[0] == '\0')
dst = "/" + dst;
SNode tmp;
if (self->m_pLocalFile->lookup(src.c_str(), tmp) >= 0)
{
//if file is local, copy directly
//note that in this case, src != dst, therefore this is a regular "cp" command, not a system replication
//TODO: check disk space
self->createDir(dst.substr(0, dst.rfind('/')));
string rhome = self->reviseSysCmdPath(self->m_strHomeDir);
string rsrc = self->reviseSysCmdPath(src);
string rdst = self->reviseSysCmdPath(dst);
system(("cp " + rhome + rsrc + " " + rhome + rdst).c_str());
// if the file has been modified during the replication, remove this replica
int type = (src == dst) ? +FileChangeType::FILE_UPDATE_REPLICA : +FileChangeType::FILE_UPDATE_NEW;
struct stat64 s;
if (stat64((self->m_strHomeDir + dst).c_str(), &s) < 0)
type = +FileChangeType::FILE_UPDATE_NO;
if (self->report(master_ip, master_port, transid, dst, type) < 0)
system(("rm " + rhome + rdst).c_str());
// clear this transaction
self->m_TransManager.updateSlave(transid, self->m_iSlaveID);
return NULL;
}
bool success = true;
queue<string> tr;
tr.push(src);
while (!tr.empty())
{
string src_path = tr.front();
tr.pop();
// try list this path
SectorMsg msg;
msg.setType(101);
msg.setKey(0);
msg.setData(0, src_path.c_str(), src_path.length() + 1);
Address addr;
self->m_Routing.lookup(src_path, addr);
if (self->m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0)
{
success = false;
break;
}
if (msg.getType() >= 0)
{
// if this is a directory, put all files and sub-drectories into the queue of files to be copied
string filelist = msg.getData();
unsigned int s = 0;
while (s < filelist.length())
{
int t = filelist.find(';', s);
SNode sn;
sn.deserialize(filelist.substr(s, t - s).c_str());
tr.push(src_path + "/" + sn.m_strName);
s = t + 1;
}
continue;
}
// open the file and copy it to local
msg.setType(110);
msg.setKey(0);
int32_t mode = SF_MODE::READ;
msg.setData(0, (char*)&mode, 4);
int64_t reserve = 0;
msg.setData(4, (char*)&reserve, 8);
int32_t localport = self->m_DataChn.getPort();
msg.setData(12, (char*)&localport, 4);
msg.setData(16, "\0", 1);
msg.setData(80, src_path.c_str(), src_path.length() + 1);
if ((self->m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0) || (msg.getType() < 0))
{
success = false;
break;
}
int32_t session = *(int32_t*)msg.getData();
int64_t size = *(int64_t*)(msg.getData() + 4);
time_t ts = *(int64_t*)(msg.getData() + 12);
string ip = msg.getData() + 24;
int32_t port = *(int32_t*)(msg.getData() + 64 + 24);
if (!self->m_DataChn.isConnected(ip, port))
{
if (self->m_DataChn.connect(ip, port) < 0)
{
success = false;
break;
}
}
// download command: 3
int32_t cmd = 3;
self->m_DataChn.send(ip, port, session, (char*)&cmd, 4);
int64_t offset = 0;
self->m_DataChn.send(ip, port, session, (char*)&offset, 8);
int response = -1;
if ((self->m_DataChn.recv4(ip, port, session, response) < 0) || (-1 == response))
{
success = false;
break;
}
string dst_path = dst;
if (src != src_path)
dst_path += "/" + src_path.substr(src.length() + 1, src_path.length() - src.length() - 1);
//copy to .tmp first, then move to real location
self->createDir(string(".tmp") + dst_path.substr(0, dst_path.rfind('/')));
fstream ofs;
ofs.open((self->m_strHomeDir + ".tmp" + dst_path).c_str(), ios::out | ios::binary | ios::trunc);
int64_t unit = 64000000; //send 64MB each time
int64_t torecv = size;
int64_t recd = 0;
while (torecv > 0)
{
int64_t block = (torecv < unit) ? torecv : unit;
if (self->m_DataChn.recvfile(ip, port, session, ofs, offset + recd, block) < 0)
{
success = false;
break;
}
recd += block;
torecv -= block;
}
ofs.close();
// update total received data size
self->m_SlaveStat.updateIO(ip, size, +SlaveStat::SYS_IN);
cmd = 5;
self->m_DataChn.send(ip, port, session, (char*)&cmd, 4);
self->m_DataChn.recv4(ip, port, session, cmd);
if (src == dst)
{
//utime: update timestamp according to the original copy, for replica only; files created by "cp" have new timestamp
utimbuf ut;
ut.actime = ts;
ut.modtime = ts;
utime((self->m_strHomeDir + ".tmp" + dst_path).c_str(), &ut);
}
}
string rhome = self->reviseSysCmdPath(self->m_strHomeDir);
string rfile = self->reviseSysCmdPath(dst);
if (success)
{
// move from temporary dir to the real dir when the copy is completed
self->createDir(dst.substr(0, dst.rfind('/')));
system(("mv " + rhome + ".tmp" + rfile + " " + rhome + rfile).c_str());
// if the file has been modified during the replication, remove this replica
int32_t type = (src == dst) ? +FileChangeType::FILE_UPDATE_REPLICA : +FileChangeType::FILE_UPDATE_NEW;
if (self->report(master_ip, master_port, transid, dst, type) < 0)
unlink((rhome + rfile).c_str());
}
else
{
// failed, remove all temporary files
system(("rm -rf " + rhome + ".tmp" + rfile).c_str());
self->report(master_ip, master_port, transid, "", +FileChangeType::FILE_UPDATE_NO);
}
// clear this transaction
self->m_TransManager.updateSlave(transid, self->m_iSlaveID);
return NULL;
}
int Index::deserialize(ifstream& ifs, map<string, SNode>& metadata, const Address* addr)
{
vector<string> dirs;
dirs.resize(1024);
map<string, SNode>* currdir = &metadata;
int currlevel = 1;
while (!ifs.eof())
{
char tmp[4096];
tmp[4095] = 0;
char* buf = tmp;
ifs.getline(buf, 4096);
int len = strlen(buf);
if ((len <= 0) || (len >= 4095))
continue;
for (int i = 0; i < len; ++ i)
{
if (buf[i] == ' ')
{
buf[i] = '\0';
break;
}
}
int level = atoi(buf);
SNode sn;
sn.deserialize(buf + strlen(buf) + 1);
if ((!sn.m_bIsDir) && (NULL != addr))
{
sn.m_sLocation.clear();
sn.m_sLocation.insert(*addr);
}
if (level == currlevel)
{
(*currdir)[sn.m_strName] = sn;
dirs[level] = sn.m_strName;
}
else if (level == currlevel + 1)
{
map<string, SNode>::iterator s = currdir->find(dirs[currlevel]);
currdir = &(s->second.m_mDirectory);
currlevel = level;
(*currdir)[sn.m_strName] = sn;
dirs[level] = sn.m_strName;
}
else if (level < currlevel)
{
currdir = &metadata;
for (int i = 1; i < level; ++ i)
{
map<string, SNode>::iterator s = currdir->find(dirs[i]);
currdir = &(s->second.m_mDirectory);
}
currlevel = level;
(*currdir)[sn.m_strName] = sn;
dirs[level] = sn.m_strName;
}
}
return 0;
}
int Index::update(const string& fileinfo, const Address& loc, const int& type)
{
CMutexGuard mg(m_MetaLock);
SNode sn;
sn.deserialize(fileinfo.c_str());
sn.m_sLocation.insert(loc);
vector<string> dir;
parsePath(sn.m_strName.c_str(), dir);
if (dir.empty())
return -1;
string filename = *(dir.rbegin());
sn.m_strName = filename;
dir.erase(dir.begin() + dir.size() - 1);
map<string, SNode>* currdir = &m_mDirectory;
map<string, SNode>::iterator s;
for (vector<string>::iterator d = dir.begin(); d != dir.end(); ++ d)
{
s = currdir->find(*d);
if (s == currdir->end())
{
if ((type == 3) || (type == 2))
{
// this is for new files only
return -1;
}
SNode n;
n.m_strName = *d;
n.m_bIsDir = true;
n.m_llTimeStamp = sn.m_llTimeStamp;
n.m_llSize = 0;
(*currdir)[*d] = n;
s = currdir->find(*d);
}
s->second.m_llTimeStamp = sn.m_llTimeStamp;
currdir = &(s->second.m_mDirectory);
}
s = currdir->find(filename);
if (s == currdir->end())
{
if ((type == 3) || (type == 2))
{
// this is for new files only
return -1;
}
(*currdir)[filename] = sn;
return 1;
}
else
{
if ((type == 1) || (type == 2))
{
// modification to an existing copy
// this maybe a new file and the master already registered the file when it is created, so it is treated as an existing file too
if ((s->second.m_llSize != sn.m_llSize) || (s->second.m_llTimeStamp != sn.m_llTimeStamp))
{
s->second.m_llSize = sn.m_llSize;
s->second.m_llTimeStamp = sn.m_llTimeStamp;
s->second.m_sLocation.insert(loc);
}
return s->second.m_sLocation.size();
}
if (type == 3)
{
// a new replica
if (s->second.m_sLocation.find(loc) != s->second.m_sLocation.end())
return -1;
if ((s->second.m_llSize != sn.m_llSize) || (s->second.m_llTimeStamp != sn.m_llTimeStamp))
return -1;
s->second.m_sLocation.insert(loc);
return s->second.m_sLocation.size();
}
}
return -1;
}
