void* RTSPServer::clientAcceptLoop(void *arg) {
if (pthread_detach(pthread_self())) {
perror("ERROR on pthread_detach");
return NULL;
}
RTSPServer *server = (RTSPServer*)arg;
while (true) {
sockaddr_in sa_cli;
socklen_t cli_addr_len = sizeof(sa_cli);
int connfd = accept(server->sfd, (sockaddr*)&sa_cli, &cli_addr_len);
if (connfd < 0) {
perror("ERROR on slave accept");
return NULL;
}
char clientIP[16];
inet_ntop(AF_INET, &sa_cli.sin_addr.s_addr, clientIP, sizeof(clientIP));
printf("slave %s:%d connected\n", clientIP, ntohs(sa_cli.sin_port));
Slave *slave = new Slave(connfd);
slave->setIPandPort(clientIP, ntohs(sa_cli.sin_port));
server->addSlave(connfd, slave);
pthread_t thread;
if (pthread_create(&thread, NULL, clientLoop, slave)) {
perror("ERROR on pthread_create clientLoop");
delete slave;
server->delSlave(connfd);
return NULL;
}
}
return NULL;
}
void FetchAndStoreAndExpungeAndExpunge(State* state)
{
Future<Variable<Slaves> > future1 = state->fetch<Slaves>("slaves");
AWAIT_READY(future1);
Variable<Slaves> variable = future1.get();
Slaves slaves1 = variable.get();
ASSERT_EQ(0, slaves1.slaves().size());
Slave* slave = slaves1.add_slaves();
slave->mutable_info()->set_hostname("localhost");
variable = variable.mutate(slaves1);
Future<Option<Variable<Slaves> > > future2 = state->store(variable);
AWAIT_READY(future2);
ASSERT_SOME(future2.get());
variable = future2.get().get();
Future<bool> future3 = state->expunge(variable);
AWAIT_READY(future3);
ASSERT_TRUE(future3.get());
future3 = state->expunge(variable);
AWAIT_READY(future3);
ASSERT_FALSE(future3.get());
}
void Master::addSlave(Machine slaveToAdd){
//Slave p = new Slave(slaveToAdd);
//cout<<"HERERERERERE"<<endl;
Slave newSlaveToAdd;
newSlaveToAdd.setMachineID(slaveToAdd.getMachineID());
newSlaveToAdd.setMachineIP(slaveToAdd.getMachineIP());
newSlaveToAdd.setMachineStatus(slaveToAdd.getMachineStatus());
newSlaveToAdd.setMachineType(slaveToAdd.getMachineType());
if(slaveCount != 0){
Slave temp[slaveCount];
for(int i = 0; i < slaveCount; i++){
temp[i] = masterSlaves[i];
}
masterSlaves = new Slave[slaveCount+1];
for(int j = 0; j < slaveCount; j++){
masterSlaves[j] = temp[j];
}
masterSlaves[slaveCount] = newSlaveToAdd;
}
else{
//cout<<"dont have"<<endl;
masterSlaves = new Slave[slaveCount+1];
masterSlaves[slaveCount] = newSlaveToAdd;
}
//masterSlaves[slaveCount] = slaveToAdd;
slaveCount++;
//cout<<"leaving"<<endl;
}
int main(int argc, char** argv) {
parse_args(argc, argv);
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &RANK);
MPI_Comm_size(MPI_COMM_WORLD, &WORLD_SIZE);
if (RANK == 0) {
printf("* width: %u\n", WIDTH);
printf("* height: %u\n", HEIGHT);
printf("* depth: %u\n", DEPTH);
printf("\n");
comm = new CommunicatorMaster(WORLD_SIZE - 1);
game();
delete comm;
} else {
Slave slave;
slave.run();
}
MPI_Finalize();
return 0;
}
void FetchAndStoreAndFetch(State* state)
{
Future<Variable<Slaves> > future1 = state->fetch<Slaves>("slaves");
AWAIT_READY(future1);
Variable<Slaves> variable = future1.get();
Slaves slaves1 = variable.get();
ASSERT_EQ(0, slaves1.slaves().size());
Slave* slave = slaves1.add_slaves();
slave->mutable_info()->set_hostname("localhost");
variable = variable.mutate(slaves1);
Future<Option<Variable<Slaves> > > future2 = state->store(variable);
AWAIT_READY(future2);
ASSERT_SOME(future2.get());
future1 = state->fetch<Slaves>("slaves");
AWAIT_READY(future1);
variable = future1.get();
Slaves slaves2 = variable.get();
ASSERT_EQ(1, slaves2.slaves().size());
EXPECT_EQ("localhost", slaves2.slaves(0).info().hostname());
}
int main(int argc, char** argv)
{
cout << SectorVersionString << endl;
SlaveConf global_conf;
CmdLineParser clp;
clp.parse(argc, argv);
for (map<string, string>::const_iterator i = clp.m_mDFlags.begin(); i != clp.m_mDFlags.end(); ++ i)
{
if (i->first == "mh")
global_conf.m_strMasterHost = i->second;
else if (i->first == "mp")
global_conf.m_iMasterPort = atoi(i->second.c_str());
else if (i->first == "h")
{
global_conf.m_strHomeDir = i->second;
}
else if (i->first == "ds")
global_conf.m_llMaxDataSize = atoll(i->second.c_str()) * 1024 * 1024;
else if (i->first == "log")
global_conf.m_iLogLevel = atoi(i->second.c_str());
else
{
cout << "warning: unrecognized flag " << i->first << endl;
help();
}
}
string base = "";
if (clp.m_vParams.size() == 1)
base = clp.m_vParams.front();
else if (clp.m_vParams.size() > 1)
cout << "warning: wrong parameters ignored.\n";
Slave s;
if (s.init(&base, &global_conf) < 0)
{
cout << "error: failed to initialize the slave. check slave configurations.\n";
return-1;
}
if (s.connect() < 0)
{
cout << "error: failed to connect to the master, or the connection request is rejected.\n";
return -1;
}
s.run();
s.close();
return 0;
}
Slave* Slave::holdSlave( const QString &protocol, const KURL& url )
{
//kdDebug(7002) << "holdSlave '" << protocol << "' for " << url.prettyURL() << endl;
// Firstly take into account all special slaves
if (protocol == "data")
return 0;
DCOPClient *client = kapp->dcopClient();
if (!client->isAttached())
client->attach();
QString prefix = locateLocal("socket", KGlobal::instance()->instanceName());
KTempFile socketfile(prefix, QString::fromLatin1(".slave-socket"));
if ( socketfile.status() != 0 )
return 0;
#ifdef __CYGWIN__
socketfile.close();
socketfile.unlink();
#endif
#ifndef Q_WS_WIN
KServerSocket *kss = new KServerSocket(QFile::encodeName(socketfile.name()));
Slave *slave = new Slave(kss, protocol, socketfile.name());
#else
Slave *slave = 0;
#endif
QByteArray params, reply;
QCString replyType;
QDataStream stream(params, IO_WriteOnly);
stream << url << socketfile.name();
QCString launcher = KApplication::launcher();
if (!client->call(launcher, launcher, "requestHoldSlave(KURL,QString)",
params, replyType, reply)) {
delete slave;
return 0;
}
QDataStream stream2(reply, IO_ReadOnly);
pid_t pid;
stream2 >> pid;
if (!pid)
{
delete slave;
return 0;
}
#ifndef Q_WS_WIN
slave->setPID(pid);
QTimer::singleShot(1000*SLAVE_CONNECTION_TIMEOUT_MIN, slave, SLOT(timeout()));
#endif
return slave;
}
// Listing 0 code/ch10
int ACE_TMAIN (int argc, ACE_TCHAR *argv[])
{
if (argc > 1) // Slave mode
{
Slave s;
return s.doWork ();
}
// Else, Master mode
Manager m (argv[0]);
return m.doWork ();
}
void Slave::LoadRDBRoutine(void* cb)
{
Slave* slave = (Slave*) cb;
if (NULL != slave->m_client)
{
slave->m_client->GetService().Continue();
}
if (time(NULL) - slave->m_routine_ts > 10)
{
slave->Routine();
}
}
void* RTSPServer::clientLoop(void *arg) {
if (pthread_detach(pthread_self())) {
perror("ERROR on pthread_detach");
return NULL;
}
Slave *slave = (Slave*)arg;
slave->readAndUpdate(true);
while (slave->readAndUpdate(false));
slave->disconnect();
return NULL;
}
SSDBServer::~SSDBServer(){
std::vector<Slave *>::iterator it;
for(it = slaves.begin(); it != slaves.end(); it++){
Slave *slave = *it;
slave->stop();
delete slave;
}
delete backend_dump;
delete backend_sync;
delete expiration;
delete cluster;
log_debug("SSDBServer finalized");
}
int main(int argc, char** argv)
{
Slave s;
int res;
if (argc > 1)
res = s.init(argv[1]);
else
res = s.init();
if (res < 0)
return -1;
if (s.connect() < 0)
return -1;
s.run();
return 1;
}
SSDB::~SSDB(){
for(std::vector<Slave *>::iterator it = slaves.begin(); it != slaves.end(); it++){
Slave *slave = *it;
slave->stop();
delete slave;
}
if(binlogs){
delete binlogs;
}
if(db){
delete db;
}
if(options.block_cache){
delete options.block_cache;
}
if(options.filter_policy){
delete options.filter_policy;
}
if(meta_db){
delete meta_db;
}
log_debug("SSDB finalized");
}
void Names(State* state)
{
Future<Variable<Slaves> > future1 = state->fetch<Slaves>("slaves");
AWAIT_READY(future1);
Variable<Slaves> variable = future1.get();
Slaves slaves1 = variable.get();
ASSERT_EQ(0, slaves1.slaves().size());
Slave* slave = slaves1.add_slaves();
slave->mutable_info()->set_hostname("localhost");
variable = variable.mutate(slaves1);
Future<Option<Variable<Slaves> > > future2 = state->store(variable);
AWAIT_READY(future2);
ASSERT_SOME(future2.get());
Future<set<string> > names = state->names();
AWAIT_READY(names);
ASSERT_EQ(1u, names.get().size());
EXPECT_NE(names.get().find("slaves"), names.get().end());
}
TEST_F(LogStateTest, Timeout)
{
Clock::pause();
Future<Variable<Slaves> > future1 = state->fetch<Slaves>("slaves");
AWAIT_READY(future1);
Variable<Slaves> variable = future1.get();
Slaves slaves1 = variable.get();
ASSERT_EQ(0, slaves1.slaves().size());
Slave* slave = slaves1.add_slaves();
slave->mutable_info()->set_hostname("localhost");
variable = variable.mutate(slaves1);
// Now terminate the replica so the store will timeout.
terminate(replica2->pid());
wait(replica2->pid());
Future<Option<Variable<Slaves> > > future2 = state->store(variable);
Future<Option<Variable<Slaves> > > future3 =
future2.after(Seconds(5), lambda::bind(&timeout, lambda::_1));
ASSERT_TRUE(future2.isPending());
ASSERT_TRUE(future3.isPending());
Clock::advance(Seconds(5));
AWAIT_DISCARDED(future2);
AWAIT_FAILED(future3);
Clock::resume();
}
int proc_info(NetworkServer *net, Link *link, const Request &req,
Response *resp){
Slave *slave = (Slave *)net->data;
resp->push_back("ok");
resp->push_back("info");
resp->push_back(slave->name());
resp->push_back("links");
resp->add(net->link_count);
int64_t calls = 0;
for(proc_map_t::iterator it = net->proc_map.begin();
it != net->proc_map.end(); it++){
Command *cmd = it->second;
calls += cmd->calls;
}
resp->push_back("total_calls");
resp->add(calls);
resp->push_back("master");
resp->push_back(slave->master->name());
return 0;
}
//private slot
void SlaveKeeper::grimReaper()
{
QMultiHash<QString, Slave *>::Iterator it = m_idleSlaves.begin();
while (it != m_idleSlaves.end()) {
Slave *slave = it.value();
if (slave->idleTime() >= s_idleSlaveLifetime) {
it = m_idleSlaves.erase(it);
if (slave->job()) {
kDebug (7006) << "Idle slave" << slave << "still has job" << slave->job();
}
slave->kill();
// avoid invoking slotSlaveDied() because its cleanup services are not needed
slave->deref();
} else {
++it;
}
}
if (!m_idleSlaves.isEmpty()) {
scheduleGrimReaper();
}
}
void* Slave::_run_thread(void *arg){
Slave *slave = (Slave *)arg;
const std::vector<Bytes> *req;
Fdevents select;
const Fdevents::events_t *events;
int idle = 0;
bool reconnect = false;
#define RECV_TIMEOUT 200
#define MAX_RECV_TIMEOUT 300 * 1000
#define MAX_RECV_IDLE MAX_RECV_TIMEOUT/RECV_TIMEOUT
while(!slave->thread_quit){
if(reconnect){
slave->status = DISCONNECTED;
reconnect = false;
select.del(slave->link->fd());
delete slave->link;
slave->link = NULL;
sleep(1);
}
if(!slave->connected()){
if(slave->connect() != 1){
usleep(100 * 1000);
}else{
select.set(slave->link->fd(), FDEVENT_IN, 0, NULL);
}
continue;
}
events = select.wait(RECV_TIMEOUT);
if(events == NULL){
log_error("events.wait error: %s", strerror(errno));
sleep(1);
continue;
}else if(events->empty()){
if(idle++ >= MAX_RECV_IDLE){
log_error("the master hasn't responsed for awhile, reconnect...");
idle = 0;
reconnect = true;
}
continue;
}
idle = 0;
if(slave->link->read() <= 0){
log_error("link.read error: %s, reconnecting to master", strerror(errno));
reconnect = true;
continue;
}
while(1){
req = slave->link->recv();
if(req == NULL){
log_error("link.recv error: %s, reconnecting to master", strerror(errno));
reconnect = true;
break;
}else if(req->empty()){
break;
}else if(req->at(0) == "noauth"){
log_error("authentication required");
reconnect = true;
sleep(1);
break;
}else{
if(slave->proc(*req) == -1){
goto err;
}
}
}
} // end while
log_info("Slave thread quit");
return (void *)NULL;
err:
log_fatal("Slave thread exit unexpectedly");
exit(0);
return (void *)NULL;;
}
TEST_F(LogStateTest, Diff)
{
Future<Variable<Slaves>> future1 = state->fetch<Slaves>("slaves");
AWAIT_READY(future1);
Variable<Slaves> variable = future1.get();
Slaves slaves = variable.get();
ASSERT_EQ(0, slaves.slaves().size());
for (size_t i = 0; i < 1024; i++) {
Slave* slave = slaves.add_slaves();
slave->mutable_info()->set_hostname("localhost" + stringify(i));
}
variable = variable.mutate(slaves);
Future<Option<Variable<Slaves>>> future2 = state->store(variable);
AWAIT_READY(future2);
ASSERT_SOME(future2.get());
variable = future2.get().get();
Slave* slave = slaves.add_slaves();
slave->mutable_info()->set_hostname("localhost1024");
variable = variable.mutate(slaves);
future2 = state->store(variable);
AWAIT_READY(future2);
ASSERT_SOME(future2.get());
// It's possible that we're doing truncation asynchronously which
// will cause the test to fail because we'll end up getting a
// pending position from Log::Reader::ending which will cause
// Log::Reader::read to fail. To remedy this, we pause the clock and
// wait for all executing processe to settle.
Clock::pause();
Clock::settle();
Clock::resume();
Log::Reader reader(log);
Future<Log::Position> beginning = reader.beginning();
Future<Log::Position> ending = reader.ending();
AWAIT_READY(beginning);
AWAIT_READY(ending);
Future<list<Log::Entry>> entries = reader.read(beginning.get(), ending.get());
AWAIT_READY(entries);
// Convert each Log::Entry to a Operation.
vector<Operation> operations;
foreach (const Log::Entry& entry, entries.get()) {
// Parse the Operation from the Log::Entry.
Operation operation;
google::protobuf::io::ArrayInputStream stream(
entry.data.data(),
entry.data.size());
ASSERT_TRUE(operation.ParseFromZeroCopyStream(&stream));
operations.push_back(operation);
}
ASSERT_EQ(2u, operations.size());
EXPECT_EQ(Operation::SNAPSHOT, operations[0].type());
EXPECT_EQ(Operation::DIFF, operations[1].type());
}
int main(int argc, char** argv)
{
GOOGLE_PROTOBUF_VERIFY_VERSION;
Configurator configurator;
Logging::registerOptions(&configurator);
Slave::registerOptions(&configurator);
// The following options are executable specific (e.g., since we
// only have one instance of libprocess per execution, we only want
// to advertise the port and ip option once, here).
configurator.addOption<int>("port", 'p', "Port to listen on", 0);
configurator.addOption<string>("ip", "IP address to listen on");
configurator.addOption<string>("isolation", "Isolation module", "process");
#ifdef MESOS_WEBUI
configurator.addOption<int>("webui_port", "Web UI port", 8081);
#endif
configurator.addOption<string>(
"master",
'm',
"May be one of:\n"
" host:port\n"
" zk://host1:port1,host2:port2,.../path\n"
" zk://username:password@host1:port1,host2:port2,.../path\n"
" file://path/to/file (where file contains one of the above)");
if (argc == 2 && string("--help") == argv[1]) {
usage(argv[0], configurator);
exit(1);
}
Configuration conf;
try {
conf = configurator.load(argc, argv);
} catch (ConfigurationException& e) {
cerr << "Configuration error: " << e.what() << endl;
exit(1);
}
Logging::init(argv[0], conf);
if (conf.contains("port")) {
setenv("LIBPROCESS_PORT", conf["port"].c_str(), 1);
}
if (conf.contains("ip")) {
setenv("LIBPROCESS_IP", conf["ip"].c_str(), 1);
}
// Initialize libprocess library (but not glog, done above).
process::initialize(false);
if (!conf.contains("master")) {
cerr << "Missing required option --master (-m)" << endl;
exit(1);
}
string master = conf["master"];
string isolation = conf["isolation"];
LOG(INFO) << "Creating \"" << isolation << "\" isolation module";
IsolationModule* isolationModule = IsolationModule::create(isolation);
if (isolationModule == NULL) {
cerr << "Unrecognized isolation type: " << isolation << endl;
exit(1);
}
LOG(INFO) << "Build: " << build::DATE << " by " << build::USER;
LOG(INFO) << "Starting Mesos slave";
if (chdir(dirname(argv[0])) != 0) {
fatalerror("Could not chdir into %s", dirname(argv[0]));
}
Slave* slave = new Slave(conf, false, isolationModule);
process::spawn(slave);
Try<MasterDetector*> detector =
MasterDetector::create(master, slave->self(), false, Logging::isQuiet(conf));
CHECK(detector.isSome())
<< "Failed to create a master detector: " << detector.error();
#ifdef MESOS_WEBUI
webui::start(slave->self(), conf);
#endif
process::wait(slave->self());
delete slave;
MasterDetector::destroy(detector.get());
IsolationModule::destroy(isolationModule);
return 0;
}
void* Slave::_run_thread(void *arg){
Slave *slave = (Slave *)arg;
const SSDB *ssdb = slave->ssdb;
const char *ip = slave->master_ip.c_str();
int port = slave->master_port;
Link *link = NULL;
int retry = 0;
const std::vector<Bytes> *req;
while(true){
if(link == NULL){
if(retry){
int t = retry > 15? 15 : retry;
usleep(t * 1000 * 1000);
log_info("[%d] connecting to master at %s:%d...", retry, ip, port);
}
link = connect(ip, port, slave->next_seq, slave->last_key);
if(link == NULL){
retry ++;
continue;
}else{
retry = 0;
}
}
req = link->recv();
if(req == NULL){
retry = 1;
delete link;
link = NULL;
log_info("recv error, reconnecting to master...");
continue;
}else if(req->empty()){
if(link->read() <= 0){
retry = 1;
delete link;
link = NULL;
log_info("network error, reconnecting to master...");
}
continue;
}
Bytes cmd = req->at(0);
if(cmd == "sync_set"){
log_trace("recv sync: %s", serialize_req(*req).c_str());
if(req->size() != 4){
log_warn("invalid set params!");
break;
}
uint64_t seq = req->at(1).Uint64();
Bytes key = req->at(2);
Bytes val = req->at(3);
if(seq == 0){
// dump
slave->last_key = key.String();
}else{
// sync
slave->next_seq = seq + 1;
}
int ret = ssdb->raw_set(key, val);
if(ret == -1){
log_error("ssdb.raw_set error!");
}
slave->save_status();
}else if(cmd == "sync_del"){
log_trace("recv sync: %s", serialize_req(*req).c_str());
if(req->size() != 3){
log_warn("invalid del params!");
break;
}
uint64_t seq = req->at(1).Uint64();
Bytes key = req->at(2);
if(seq == 0){
// dump
slave->last_key = key.String();
}else{
// sync
slave->next_seq = seq + 1;
}
int ret = ssdb->raw_del(key);
if(ret == -1){
log_error("ssdb.raw_del error!");
}
slave->save_status();
}else if(cmd == "dump_end"){
log_info("dump end, step in sync");
slave->last_key = "";
}else if(cmd == "noop"){
//
}else{
log_warn("unknow sync command: %s", serialize_req(*req).c_str());
}
} // end while
if(link){
delete link;
}
log_info("Slave thread quit");
return (void *)NULL;
}
int main(int argc, char** argv)
{
GOOGLE_PROTOBUF_VERIFY_VERSION;
slave::Flags flags;
// The following flags are executable specific (e.g., since we only
// have one instance of libprocess per execution, we only want to
// advertise the IP and port option once, here).
Option<string> ip;
flags.add(&ip, "ip", "IP address to listen on");
uint16_t port;
flags.add(&port, "port", "Port to listen on", SlaveInfo().port());
string isolation;
flags.add(&isolation,
"isolation",
"Isolation mechanism, may be one of: process, cgroups",
"process");
Option<string> master;
flags.add(&master,
"master",
"May be one of:\n"
" zk://host1:port1,host2:port2,.../path\n"
" zk://username:password@host1:port1,host2:port2,.../path\n"
" file://path/to/file (where file contains one of the above)");
bool help;
flags.add(&help,
"help",
"Prints this help message",
false);
Try<Nothing> load = flags.load("MESOS_", argc, argv);
if (load.isError()) {
cerr << load.error() << endl;
usage(argv[0], flags);
exit(1);
}
if (help) {
usage(argv[0], flags);
exit(1);
}
if (master.isNone()) {
cerr << "Missing required option --master" << endl;
exit(1);
}
// Initialize libprocess.
if (ip.isSome()) {
os::setenv("LIBPROCESS_IP", ip.get());
}
os::setenv("LIBPROCESS_PORT", stringify(port));
process::initialize();
logging::initialize(argv[0], flags, true); // Catch signals.
LOG(INFO) << "Creating \"" << isolation << "\" isolator";
Isolator* isolator = Isolator::create(isolation);
if (isolator == NULL) {
cerr << "Unrecognized isolation type: " << isolation << endl;
exit(1);
}
LOG(INFO) << "Build: " << build::DATE << " by " << build::USER;
LOG(INFO) << "Starting Mesos slave";
Try<MasterDetector*> detector = MasterDetector::create(master.get());
CHECK_SOME(detector) << "Failed to create a master detector";
Files files;
Slave* slave = new Slave(flags, false, detector.get(), isolator, &files);
process::spawn(slave);
process::wait(slave->self());
delete slave;
delete detector.get();
Isolator::destroy(isolator);
return 0;
}
void* Slave::copy(void* p)
{
Slave* self = ((Param3*)p)->serv_instance;
int transid = ((Param3*)p)->transid;
string src = ((Param3*)p)->src;
string dst = ((Param3*)p)->dst;
string master_ip = ((Param3*)p)->master_ip;
int master_port = ((Param3*)p)->master_port;
delete (Param3*)p;
if (src.c_str()[0] == '\0')
src = "/" + src;
if (dst.c_str()[0] == '\0')
dst = "/" + dst;
SNode tmp;
if (self->m_pLocalFile->lookup(src.c_str(), tmp) >= 0)
{
//if file is local, copy directly
//note that in this case, src != dst, therefore this is a regular "cp" command, not a system replication
//TODO: check disk space
self->createDir(dst.substr(0, dst.rfind('/')));
string rhome = self->reviseSysCmdPath(self->m_strHomeDir);
string rsrc = self->reviseSysCmdPath(src);
string rdst = self->reviseSysCmdPath(dst);
system(("cp " + rhome + rsrc + " " + rhome + rdst).c_str());
// if the file has been modified during the replication, remove this replica
int type = (src == dst) ? +FileChangeType::FILE_UPDATE_REPLICA : +FileChangeType::FILE_UPDATE_NEW;
struct stat64 s;
if (stat64((self->m_strHomeDir + dst).c_str(), &s) < 0)
type = +FileChangeType::FILE_UPDATE_NO;
if (self->report(master_ip, master_port, transid, dst, type) < 0)
system(("rm " + rhome + rdst).c_str());
// clear this transaction
self->m_TransManager.updateSlave(transid, self->m_iSlaveID);
return NULL;
}
bool success = true;
queue<string> tr;
tr.push(src);
while (!tr.empty())
{
string src_path = tr.front();
tr.pop();
// try list this path
SectorMsg msg;
msg.setType(101);
msg.setKey(0);
msg.setData(0, src_path.c_str(), src_path.length() + 1);
Address addr;
self->m_Routing.lookup(src_path, addr);
if (self->m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0)
{
success = false;
break;
}
if (msg.getType() >= 0)
{
// if this is a directory, put all files and sub-drectories into the queue of files to be copied
string filelist = msg.getData();
unsigned int s = 0;
while (s < filelist.length())
{
int t = filelist.find(';', s);
SNode sn;
sn.deserialize(filelist.substr(s, t - s).c_str());
tr.push(src_path + "/" + sn.m_strName);
s = t + 1;
}
continue;
}
// open the file and copy it to local
msg.setType(110);
msg.setKey(0);
int32_t mode = SF_MODE::READ;
msg.setData(0, (char*)&mode, 4);
int64_t reserve = 0;
msg.setData(4, (char*)&reserve, 8);
int32_t localport = self->m_DataChn.getPort();
msg.setData(12, (char*)&localport, 4);
msg.setData(16, "\0", 1);
msg.setData(80, src_path.c_str(), src_path.length() + 1);
if ((self->m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0) || (msg.getType() < 0))
{
success = false;
break;
}
int32_t session = *(int32_t*)msg.getData();
int64_t size = *(int64_t*)(msg.getData() + 4);
time_t ts = *(int64_t*)(msg.getData() + 12);
string ip = msg.getData() + 24;
int32_t port = *(int32_t*)(msg.getData() + 64 + 24);
if (!self->m_DataChn.isConnected(ip, port))
{
if (self->m_DataChn.connect(ip, port) < 0)
{
success = false;
break;
}
}
// download command: 3
int32_t cmd = 3;
self->m_DataChn.send(ip, port, session, (char*)&cmd, 4);
int64_t offset = 0;
self->m_DataChn.send(ip, port, session, (char*)&offset, 8);
int response = -1;
if ((self->m_DataChn.recv4(ip, port, session, response) < 0) || (-1 == response))
{
success = false;
break;
}
string dst_path = dst;
if (src != src_path)
dst_path += "/" + src_path.substr(src.length() + 1, src_path.length() - src.length() - 1);
//copy to .tmp first, then move to real location
self->createDir(string(".tmp") + dst_path.substr(0, dst_path.rfind('/')));
fstream ofs;
ofs.open((self->m_strHomeDir + ".tmp" + dst_path).c_str(), ios::out | ios::binary | ios::trunc);
int64_t unit = 64000000; //send 64MB each time
int64_t torecv = size;
int64_t recd = 0;
while (torecv > 0)
{
int64_t block = (torecv < unit) ? torecv : unit;
if (self->m_DataChn.recvfile(ip, port, session, ofs, offset + recd, block) < 0)
{
success = false;
break;
}
recd += block;
torecv -= block;
}
ofs.close();
// update total received data size
self->m_SlaveStat.updateIO(ip, size, +SlaveStat::SYS_IN);
cmd = 5;
self->m_DataChn.send(ip, port, session, (char*)&cmd, 4);
self->m_DataChn.recv4(ip, port, session, cmd);
if (src == dst)
{
//utime: update timestamp according to the original copy, for replica only; files created by "cp" have new timestamp
utimbuf ut;
ut.actime = ts;
ut.modtime = ts;
utime((self->m_strHomeDir + ".tmp" + dst_path).c_str(), &ut);
}
}
string rhome = self->reviseSysCmdPath(self->m_strHomeDir);
string rfile = self->reviseSysCmdPath(dst);
if (success)
{
// move from temporary dir to the real dir when the copy is completed
self->createDir(dst.substr(0, dst.rfind('/')));
system(("mv " + rhome + ".tmp" + rfile + " " + rhome + rfile).c_str());
// if the file has been modified during the replication, remove this replica
int32_t type = (src == dst) ? +FileChangeType::FILE_UPDATE_REPLICA : +FileChangeType::FILE_UPDATE_NEW;
if (self->report(master_ip, master_port, transid, dst, type) < 0)
unlink((rhome + rfile).c_str());
}
else
{
// failed, remove all temporary files
system(("rm -rf " + rhome + ".tmp" + rfile).c_str());
self->report(master_ip, master_port, transid, "", +FileChangeType::FILE_UPDATE_NO);
}
// clear this transaction
self->m_TransManager.updateSlave(transid, self->m_iSlaveID);
return NULL;
}
int proc_info(NetworkServer *net, Link *link, const Request &req, Response *resp){
SSDBServer *serv = (SSDBServer *)net->data;
resp->push_back("ok");
resp->push_back("ssdb-server");
resp->push_back("version");
resp->push_back(SSDB_VERSION);
{
resp->push_back("links");
resp->add(net->link_count);
}
{
int64_t calls = 0;
proc_map_t::iterator it;
for(it=net->proc_map.begin(); it!=net->proc_map.end(); it++){
Command *cmd = it->second;
calls += cmd->calls;
}
resp->push_back("total_calls");
resp->add(calls);
}
{
uint64_t size = serv->ssdb->size();
resp->push_back("dbsize");
resp->push_back(str(size));
}
{
std::string s = serv->ssdb->binlogs->stats();
resp->push_back("binlogs");
resp->push_back(s);
}
{
std::vector<std::string> syncs = serv->backend_sync->stats();
std::vector<std::string>::iterator it;
for(it = syncs.begin(); it != syncs.end(); it++){
std::string s = *it;
resp->push_back("replication");
resp->push_back(s);
}
}
{
std::vector<Slave *>::iterator it;
for(it = serv->slaves.begin(); it != serv->slaves.end(); it++){
Slave *slave = *it;
std::string s = slave->stats();
resp->push_back("replication");
resp->push_back(s);
}
}
{
std::string val;
std::string s, e;
serv->get_kv_range(&s, &e);
char buf[512];
{
snprintf(buf, sizeof(buf), " kv : \"%s\" - \"%s\"",
str_escape(s).c_str(),
str_escape(e).c_str()
);
val.append(buf);
}
{
snprintf(buf, sizeof(buf), "\n hash: \"\" - \"\"");
val.append(buf);
}
{
snprintf(buf, sizeof(buf), "\n zset: \"\" - \"\"");
val.append(buf);
}
{
snprintf(buf, sizeof(buf), "\n list: \"\" - \"\"");
val.append(buf);
}
resp->push_back("serv_key_range");
resp->push_back(val);
}
if(req.size() == 1 || req[1] == "range"){
std::string val;
std::vector<std::string> tmp;
int ret = serv->ssdb->key_range(&tmp);
if(ret == 0){
char buf[512];
snprintf(buf, sizeof(buf), " kv : \"%s\" - \"%s\"",
hexmem(tmp[0].data(), tmp[0].size()).c_str(),
hexmem(tmp[1].data(), tmp[1].size()).c_str()
);
val.append(buf);
snprintf(buf, sizeof(buf), "\n hash: \"%s\" - \"%s\"",
hexmem(tmp[2].data(), tmp[2].size()).c_str(),
hexmem(tmp[3].data(), tmp[3].size()).c_str()
);
val.append(buf);
snprintf(buf, sizeof(buf), "\n zset: \"%s\" - \"%s\"",
hexmem(tmp[4].data(), tmp[4].size()).c_str(),
hexmem(tmp[5].data(), tmp[5].size()).c_str()
);
val.append(buf);
snprintf(buf, sizeof(buf), "\n list: \"%s\" - \"%s\"",
hexmem(tmp[6].data(), tmp[6].size()).c_str(),
hexmem(tmp[7].data(), tmp[7].size()).c_str()
);
val.append(buf);
}
resp->push_back("data_key_range");
resp->push_back(val);
}
if(req.size() == 1 || req[1] == "leveldb"){
std::vector<std::string> tmp = serv->ssdb->info();
for(int i=0; i<(int)tmp.size(); i++){
std::string block = tmp[i];
resp->push_back(block);
}
}
if(req.size() > 1 && req[1] == "cmd"){
proc_map_t::iterator it;
for(it=net->proc_map.begin(); it!=net->proc_map.end(); it++){
Command *cmd = it->second;
resp->push_back("cmd." + cmd->name);
char buf[128];
snprintf(buf, sizeof(buf), "calls: %" PRIu64 "\ttime_wait: %.0f\ttime_proc: %.0f",
cmd->calls, cmd->time_wait, cmd->time_proc);
resp->push_back(buf);
}
}
return 0;
}
DWORD WINAPI Slave::SPEShuffler(LPVOID p)
#endif
{
Slave* self = ((Param5*)p)->serv_instance;
int transid = ((Param5*)p)->transid;
string client_ip = ((Param5*)p)->client_ip;
int client_port = ((Param5*)p)->client_ctrl_port;
int client_data_port = ((Param5*)p)->client_data_port;
string path = ((Param5*)p)->path;
string localfile = ((Param5*)p)->filename;
int bucketnum = ((Param5*)p)->bucketnum;
CGMP* gmp = ((Param5*)p)->gmp;
string function = ((Param5*)p)->function;
int bucketid = ((Param5*)p)->bucketid;
const int key = ((Param5*)p)->key;
const int type = ((Param5*)p)->type;
string master_ip = ((Param5*)p)->master_ip;
int master_port = ((Param5*)p)->master_port;
queue<Bucket>* bq = NULL;
CMutex* bqlock = NULL;
CCond* bqcond = NULL;
int64_t* pendingSize = NULL;
pthread_t shufflerex;
bool init_success = true;
//set up data connection, for keep-alive purpose
if (self->m_DataChn.connect(client_ip, client_data_port) < 0)
{
init_success = false;
}
else
{
// read library files for MapReduce, no need for Sphere UDF
if (type == 1)
self->acceptLibrary(key, client_ip, client_data_port, transid);
bq = new queue<Bucket>;
bqlock = new CMutex;
bqcond = new CCond;
pendingSize = new int64_t;
*pendingSize = 0;
((Param5*)p)->bq = bq;
((Param5*)p)->bqlock = bqlock;
((Param5*)p)->bqcond = bqcond;
((Param5*)p)->pending = pendingSize;
#ifndef WIN32
pthread_create(&shufflerex, NULL, SPEShufflerEx, p);
#else
DWORD ThreadID;
shufflerex = CreateThread(NULL, 0, SPEShufflerEx, p, NULL, &ThreadID);
#endif
self->m_SectorLog << LogStart(LogLevel::SCREEN) << "SPE Shuffler " << path << " " << localfile << " " << bucketnum << LogEnd();
}
while (init_success)
{
string speip;
int speport;
SectorMsg msg;
int msgid;
int r = gmp->recvfrom(speip, speport, msgid, &msg, false);
// client releases the task or client has already been shutdown
if (((r > 0) && (speip == client_ip) && (speport == client_port))
|| ((r < 0) && (!self->m_DataChn.isConnected(client_ip, client_data_port))))
{
Bucket b;
b.totalnum = -1;
b.totalsize = 0;
bqlock->acquire();
bq->push(b);
bqcond->signal();
bqlock->release();
break;
}
if (r < 0)
continue;
if (*pendingSize > 256000000)
{
// too many incoming results, ask the sender to wait
// the receiver buffer size threshold is set to 256MB. This prevents the shuffler from being overflowed
// it also helps direct the traffic to less congested shuffler and leads to better load balance
msg.setType(-msg.getType());
gmp->sendto(speip, speport, msgid, &msg);
}
else
{
Bucket b;
b.totalnum = *(int32_t*)(msg.getData() + 8);;
b.totalsize = *(int32_t*)(msg.getData() + 12);
b.src_ip = speip;
b.src_dataport = *(int32_t*)msg.getData();
b.session = *(int32_t*)(msg.getData() + 4);
gmp->sendto(speip, speport, msgid, &msg);
if (!self->m_DataChn.isConnected(speip, b.src_dataport))
self->m_DataChn.connect(speip, b.src_dataport);
bqlock->acquire();
bq->push(b);
*pendingSize += b.totalsize;
bqcond->signal();
bqlock->release();
}
}
if (init_success)
{
#ifndef WIN32
pthread_join(shufflerex, NULL);
#else
WaitForSingleObject(shufflerex, INFINITE);
#endif
delete bqlock;
delete bqcond;
delete pendingSize;
SectorMsg msg;
msg.setType(1); // success, return result
msg.setData(0, (char*)&(bucketid), 4);
int progress = 100;
msg.setData(4, (char*)&progress, 4);
msg.m_iDataLength = SectorMsg::m_iHdrSize + 8;
int id = 0;
self->m_GMP.sendto(client_ip.c_str(), client_port, id, &msg);
self->m_SectorLog << LogStart(LogLevel::LEVEL_3) << "bucket completed 100 " << client_ip << " " << client_port << LogEnd();
}
gmp->close();
delete gmp;
self->reportSphere(master_ip, master_port, transid);
// clear this transaction
self->m_TransManager.updateSlave(transid, self->m_iSlaveID);
return NULL;
}
DWORD WINAPI Slave::SPEShufflerEx(LPVOID p)
#endif
{
Slave* self = ((Param5*)p)->serv_instance;
int transid = ((Param5*)p)->transid;
string path = ((Param5*)p)->path;
string localfile = ((Param5*)p)->filename;
int bucketnum = ((Param5*)p)->bucketnum;
const int key = ((Param5*)p)->key;
const int type = ((Param5*)p)->type;
string function = ((Param5*)p)->function;
string master_ip = ((Param5*)p)->master_ip;
int master_port = ((Param5*)p)->master_port;
queue<Bucket>* bq = ((Param5*)p)->bq;
CMutex* bqlock = ((Param5*)p)->bqlock;
CCond* bqcond = ((Param5*)p)->bqcond;
int64_t* pendingSize = ((Param5*)p)->pending;
delete (Param5*)p;
self->createDir(path);
// remove old result data files
for (int i = 0; i < bucketnum; ++ i)
{
int size = self->m_strHomeDir.length() + path.length() + localfile.length() + 64;
char* tmp = new char[size];
snprintf(tmp, size, "%s.%d", (self->m_strHomeDir + path + "/" + localfile).c_str(), i);
LocalFS::erase(tmp);
snprintf(tmp, size, "%s.%d.idx", (self->m_strHomeDir + path + "/" + localfile).c_str(), i);
LocalFS::erase(tmp);
delete [] tmp;
}
// index file initial offset
vector<int64_t> offset;
offset.resize(bucketnum);
for (vector<int64_t>::iterator i = offset.begin(); i != offset.end(); ++ i)
*i = 0;
set<int> fileid;
while (true)
{
bqlock->acquire();
while (bq->empty())
bqcond->wait(*bqlock);
Bucket b = bq->front();
bq->pop();
*pendingSize -= b.totalsize;
bqlock->release();
if (b.totalnum == -1)
break;
string speip = b.src_ip;
int dataport = b.src_dataport;
int session = b.session;
for (int i = 0; i < b.totalnum; ++ i)
{
int bucket = 0;
if (self->m_DataChn.recv4(speip, dataport, session, bucket) < 0)
continue;
fileid.insert(bucket);
char* tmp = new char[self->m_strHomeDir.length() + path.length() + localfile.length() + 64];
sprintf(tmp, "%s.%d", (self->m_strHomeDir + path + "/" + localfile).c_str(), bucket);
fstream datafile(tmp, ios::out | ios::binary | ios::app);
sprintf(tmp, "%s.%d.idx", (self->m_strHomeDir + path + "/" + localfile).c_str(), bucket);
fstream indexfile(tmp, ios::out | ios::binary | ios::app);
delete [] tmp;
int64_t start = offset[bucket];
if (0 == start)
indexfile.write((char*)&start, 8);
int32_t len;
char* data = NULL;
if (self->m_DataChn.recv(speip, dataport, session, data, len) < 0)
continue;
datafile.write(data, len);
delete [] data;
tmp = NULL;
if (self->m_DataChn.recv(speip, dataport, session, tmp, len) < 0)
continue;
int64_t* index = (int64_t*)tmp;
for (int j = 0; j < len / 8; ++ j)
index[j] += start;
offset[bucket] = index[len / 8 - 1];
indexfile.write(tmp, len);
delete [] tmp;
datafile.close();
indexfile.close();
}
// update total received data
self->m_SlaveStat.updateIO(speip, b.totalsize, +SlaveStat::SYS_IN);
}
// sort and reduce
if (type == 1)
{
void* lh = NULL;
self->openLibrary(key, function, lh);
if (NULL != lh)
{
MR_COMPARE comp = NULL;
MR_REDUCE reduce = NULL;
self->getReduceFunc(lh, function, comp, reduce);
if (NULL != comp)
{
char* tmp = new char[self->m_strHomeDir.length() + path.length() + localfile.length() + 64];
for (set<int>::iterator i = fileid.begin(); i != fileid.end(); ++ i)
{
sprintf(tmp, "%s.%d", (self->m_strHomeDir + path + "/" + localfile).c_str(), *i);
self->sort(tmp, comp, reduce);
}
delete [] tmp;
}
self->closeLibrary(lh);
}
}
// report sphere output files
char* tmp = new char[path.length() + localfile.length() + 64];
vector<string> filelist;
for (set<int>::iterator i = fileid.begin(); i != fileid.end(); ++ i)
{
sprintf(tmp, "%s.%d", (path + "/" + localfile).c_str(), *i);
filelist.push_back(tmp);
sprintf(tmp, "%s.%d.idx", (path + "/" + localfile).c_str(), *i);
filelist.push_back(tmp);
}
delete [] tmp;
self->report(master_ip, master_port, transid, filelist, 1);
return NULL;
}
DWORD WINAPI Slave::fileHandler(LPVOID p)
#endif
{
Slave* self = ((Param2*)p)->serv_instance;
string filename = self->m_strHomeDir + ((Param2*)p)->filename;
string sname = ((Param2*)p)->filename;
int key = ((Param2*)p)->key;
int mode = ((Param2*)p)->mode;
int transid = ((Param2*)p)->transid;
string client_ip = ((Param2*)p)->client_ip;
int client_port = ((Param2*)p)->client_port;
unsigned char crypto_key[16];
unsigned char crypto_iv[8];
memcpy(crypto_key, ((Param2*)p)->crypto_key, 16);
memcpy(crypto_iv, ((Param2*)p)->crypto_iv, 8);
string master_ip = ((Param2*)p)->master_ip;
int master_port = ((Param2*)p)->master_port;
delete (Param2*)p;
// uplink and downlink addresses for write, no need for read
string src_ip = client_ip;
int src_port = client_port;
string dst_ip;
int dst_port = -1;
// IO permissions
bool bRead = mode & 1;
bool bWrite = mode & 2;
bool trunc = mode & 4;
bool bSecure = mode & 16;
bool m_bChange = false;
int last_timestamp = 0;
self->m_SectorLog << LogStart(LogLevel::LEVEL_3) << "connecting to " << client_ip << " " << client_port << " " << filename << LogEnd();
if ((!self->m_DataChn.isConnected(client_ip, client_port)) && (self->m_DataChn.connect(client_ip, client_port) < 0))
{
self->m_SectorLog << LogStart(LogLevel::LEVEL_2) << "failed to connect to file client " << client_ip << " " << client_port << " " << filename << LogEnd();
// release transactions and file locks
self->m_TransManager.updateSlave(transid, self->m_iSlaveID);
self->m_pLocalFile->unlock(sname, key, mode);
self->report(master_ip, master_port, transid, sname, +FileChangeType::FILE_UPDATE_NO);
return NULL;
}
Crypto* encoder = NULL;
Crypto* decoder = NULL;
if (bSecure)
{
encoder = new Crypto;
encoder->initEnc(crypto_key, crypto_iv);
decoder = new Crypto;
decoder->initDec(crypto_key, crypto_iv);
}
//create a new directory or file in case it does not exist
if (bWrite)
{
self->createDir(sname.substr(0, sname.rfind('/')));
SNode s;
if (LocalFS::stat(filename, s) < 0)
{
ofstream newfile(filename.c_str(), ios::out | ios::binary | ios::trunc);
newfile.close();
}
}
timeval t1, t2;
gettimeofday(&t1, 0);
int64_t rb = 0;
int64_t wb = 0;
WriteLog writelog;
fstream fhandle;
if (!trunc)
fhandle.open(filename.c_str(), ios::in | ios::out | ios::binary);
else
fhandle.open(filename.c_str(), ios::in | ios::out | ios::binary | ios::trunc);
// a file session is successful only if the client issue a close() request
bool success = true;
bool run = true;
int32_t cmd = 0;
while (run)
{
if (self->m_DataChn.recv4(client_ip, client_port, transid, cmd) < 0)
break;
switch (cmd)
{
case 1: // read
{
char* param = NULL;
int tmp = 8 * 2;
if (self->m_DataChn.recv(client_ip, client_port, transid, param, tmp) < 0)
{
success = false;
break;
}
int64_t offset = *(int64_t*)param;
int64_t size = *(int64_t*)(param + 8);
delete [] param;
int32_t response = bRead ? 0 : -1;
if (fhandle.fail() || !success || !self->m_bDiskHealth || !self->m_bNetworkHealth)
response = -1;
if (self->m_DataChn.send(client_ip, client_port, transid, (char*)&response, 4) < 0)
break;
if (response == -1)
break;
if (self->m_DataChn.sendfile(client_ip, client_port, transid, fhandle, offset, size, encoder) < 0)
success = false;
else
rb += size;
// update total sent data size
self->m_SlaveStat.updateIO(client_ip, param[1], (key == 0) ? +SlaveStat::SYS_OUT : +SlaveStat::CLI_OUT);
break;
}
case 2: // write
{
if (!bWrite)
{
// if the client does not have write permission, disconnect it immediately
success = false;
break;
}
//receive offset and size information from uplink
char* param = NULL;
int tmp = 8 * 2;
if (self->m_DataChn.recv(src_ip, src_port, transid, param, tmp) < 0)
break;
int64_t offset = *(int64_t*)param;
int64_t size = *(int64_t*)(param + 8);
delete [] param;
// no secure transfer between two slaves
Crypto* tmp_decoder = decoder;
if ((client_ip != src_ip) || (client_port != src_port))
tmp_decoder = NULL;
bool io_status = (size > 0);
if (!io_status || (self->m_DataChn.recvfile(src_ip, src_port, transid, fhandle, offset, size, tmp_decoder) < size))
io_status = false;
//TODO: send incomplete write to next slave on chain, rather than -1
if (dst_port > 0)
{
// send offset and size parameters
char req[16];
*(int64_t*)req = offset;
if (io_status)
*(int64_t*)(req + 8) = size;
else
*(int64_t*)(req + 8) = -1;
self->m_DataChn.send(dst_ip, dst_port, transid, req, 16);
// send the data to the next replica in the chain
if (size > 0)
self->m_DataChn.sendfile(dst_ip, dst_port, transid, fhandle, offset, size);
}
if (!io_status)
break;
wb += size;
// update total received data size
self->m_SlaveStat.updateIO(src_ip, size, (key == 0) ? +SlaveStat::SYS_IN : +SlaveStat::CLI_IN);
// update write log
writelog.insert(offset, size);
m_bChange = true;
break;
}
case 3: // download
{
int64_t offset;
if (self->m_DataChn.recv8(client_ip, client_port, transid, offset) < 0)
{
success = false;
break;
}
int32_t response = bRead ? 0 : -1;
if (fhandle.fail() || !success || !self->m_bDiskHealth || !self->m_bNetworkHealth)
response = -1;
if (self->m_DataChn.send(client_ip, client_port, transid, (char*)&response, 4) < 0)
break;
if (response == -1)
break;
fhandle.seekg(0, ios::end);
int64_t size = (int64_t)(fhandle.tellg());
fhandle.seekg(0, ios::beg);
size -= offset;
int64_t unit = 64000000; //send 64MB each time
int64_t tosend = size;
int64_t sent = 0;
while (tosend > 0)
{
int64_t block = (tosend < unit) ? tosend : unit;
if (self->m_DataChn.sendfile(client_ip, client_port, transid, fhandle, offset + sent, block, encoder) < 0)
{
success = false;
break;
}
sent += block;
tosend -= block;
}
rb += sent;
// update total sent data size
self->m_SlaveStat.updateIO(client_ip, size, (key == 0) ? +SlaveStat::SYS_OUT : +SlaveStat::CLI_OUT);
break;
}
case 4: // upload
{
if (!bWrite)
{
// if the client does not have write permission, disconnect it immediately
success = false;
break;
}
int64_t offset = 0;
int64_t size;
if (self->m_DataChn.recv8(client_ip, client_port, transid, size) < 0)
{
success = false;
break;
}
//TODO: check available size
int32_t response = 0;
if (fhandle.fail() || !success || !self->m_bDiskHealth || !self->m_bNetworkHealth)
response = -1;
if (self->m_DataChn.send(client_ip, client_port, transid, (char*)&response, 4) < 0)
break;
if (response == -1)
break;
int64_t unit = 64000000; //send 64MB each time
int64_t torecv = size;
int64_t recd = 0;
// no secure transfer between two slaves
Crypto* tmp_decoder = decoder;
if ((client_ip != src_ip) || (client_port != src_port))
tmp_decoder = NULL;
while (torecv > 0)
{
int64_t block = (torecv < unit) ? torecv : unit;
if (self->m_DataChn.recvfile(src_ip, src_port, transid, fhandle, offset + recd, block, tmp_decoder) < 0)
{
success = false;
break;
}
if (dst_port > 0)
{
// write to uplink for next replica in the chain
if (self->m_DataChn.sendfile(dst_ip, dst_port, transid, fhandle, offset + recd, block) < 0)
break;
}
recd += block;
torecv -= block;
}
wb += recd;
// update total received data size
self->m_SlaveStat.updateIO(src_ip, size, (key == 0) ? +SlaveStat::SYS_IN : +SlaveStat::CLI_IN);
// update write log
writelog.insert(0, size);
m_bChange = true;
break;
}
case 5: // end session
// the file has been successfully closed
run = false;
break;
case 6: // read file path for local IO optimization
self->m_DataChn.send(client_ip, client_port, transid, self->m_strHomeDir.c_str(), self->m_strHomeDir.length() + 1);
break;
case 7: // synchronize with the client, make sure write is correct
{
//TODO: merge all three recv() to one
int32_t size = 0;
if (self->m_DataChn.recv4(client_ip, client_port, transid, size) < 0)
break;
char* buf = NULL;
if (self->m_DataChn.recv(client_ip, client_port, transid, buf, size) < 0)
break;
last_timestamp = 0;
if (self->m_DataChn.recv4(client_ip, client_port, transid, last_timestamp) < 0)
break;
WriteLog log;
log.deserialize(buf, size);
delete [] buf;
int32_t confirm = -1;
if (writelog.compare(log))
confirm = 1;
writelog.clear();
if (confirm > 0)
{
//synchronize timestamp
utimbuf ut;
ut.actime = last_timestamp;
ut.modtime = last_timestamp;
utime(filename.c_str(), &ut);
}
self->m_DataChn.send(client_ip, client_port, transid, (char*)&confirm, 4);
break;
}
case 8: // specify up and down links
{
char* buf = NULL;
int size = 136;
if (self->m_DataChn.recv(client_ip, client_port, transid, buf, size) < 0)
break;
int32_t response = bWrite ? 0 : -1;
if (fhandle.fail() || !success || !self->m_bDiskHealth || !self->m_bNetworkHealth)
response = -1;
if (self->m_DataChn.send(client_ip, client_port, transid, (char*)&response, 4) < 0)
break;
if (response == -1)
break;
src_ip = buf;
src_port = *(int32_t*)(buf + 64);
dst_ip = buf + 68;
dst_port = *(int32_t*)(buf + 132);
delete [] buf;
if (src_port > 0)
{
// connect to uplink in the write chain
if (!self->m_DataChn.isConnected(src_ip, src_port))
self->m_DataChn.connect(src_ip, src_port);
}
else
{
// first node in the chain, read from client
src_ip = client_ip;
src_port = client_port;
}
if (dst_port > 0)
{
//connect downlink in the write chain
if (!self->m_DataChn.isConnected(dst_ip, dst_port))
self->m_DataChn.connect(dst_ip, dst_port);
}
break;
}
default:
break;
}
}
// close local file
fhandle.close();
// update final timestamp
if (last_timestamp > 0)
{
utimbuf ut;
ut.actime = last_timestamp;
ut.modtime = last_timestamp;
utime(filename.c_str(), &ut);
}
gettimeofday(&t2, 0);
int duration = t2.tv_sec - t1.tv_sec;
double avgRS = 0;
double avgWS = 0;
if (duration > 0)
{
avgRS = rb / duration * 8.0 / 1000000.0;
avgWS = wb / duration * 8.0 / 1000000.0;
}
self->m_SectorLog << LogStart(LogLevel::LEVEL_3) << "file server closed " << src_ip << " " << src_port << " " << (long long)avgWS << " " << (long long)avgRS << LogEnd();
// clear this transaction
self->m_TransManager.updateSlave(transid, self->m_iSlaveID);
// unlock the file
// this must be done before the client is disconnected, otherwise if the client immediately re-open the file, the lock may not be released yet
self->m_pLocalFile->unlock(sname, key, mode);
// report to master the task is completed
// this also must be done before the client is disconnected, otherwise client may not be able to immediately re-open the file as the master is not updated
int change = m_bChange ? +FileChangeType::FILE_UPDATE_WRITE : +FileChangeType::FILE_UPDATE_NO;
self->report(master_ip, master_port, transid, sname, change);
if (bSecure)
{
encoder->release();
delete encoder;
decoder->release();
delete decoder;
}
if (success)
self->m_DataChn.send(client_ip, client_port, transid, (char*)&cmd, 4);
else
self->m_DataChn.sendError(client_ip, client_port, transid);
return NULL;
}
TEST_F(LogStateTest, Diff)
{
Future<Variable<Slaves>> future1 = state->fetch<Slaves>("slaves");
AWAIT_READY(future1);
Variable<Slaves> variable = future1.get();
Slaves slaves = variable.get();
ASSERT_EQ(0, slaves.slaves().size());
for (size_t i = 0; i < 1024; i++) {
Slave* slave = slaves.add_slaves();
slave->mutable_info()->set_hostname("localhost" + stringify(i));
}
variable = variable.mutate(slaves);
Future<Option<Variable<Slaves>>> future2 = state->store(variable);
AWAIT_READY(future2);
ASSERT_SOME(future2.get());
variable = future2.get().get();
Slave* slave = slaves.add_slaves();
slave->mutable_info()->set_hostname("localhost1024");
variable = variable.mutate(slaves);
future2 = state->store(variable);
AWAIT_READY(future2);
ASSERT_SOME(future2.get());
Log::Reader reader(log);
Future<Log::Position> beginning = reader.beginning();
Future<Log::Position> ending = reader.ending();
AWAIT_READY(beginning);
AWAIT_READY(ending);
Future<list<Log::Entry>> entries = reader.read(beginning.get(), ending.get());
AWAIT_READY(entries);
// Convert each Log::Entry to a Operation.
vector<Operation> operations;
foreach (const Log::Entry& entry, entries.get()) {
// Parse the Operation from the Log::Entry.
Operation operation;
google::protobuf::io::ArrayInputStream stream(
entry.data.data(),
entry.data.size());
ASSERT_TRUE(operation.ParseFromZeroCopyStream(&stream));
operations.push_back(operation);
}
ASSERT_EQ(2u, operations.size());
EXPECT_EQ(Operation::SNAPSHOT, operations[0].type());
EXPECT_EQ(Operation::DIFF, operations[1].type());
}
DWORD WINAPI Slave::copy(LPVOID p)
#endif
{
Slave* self = ((Param3*)p)->serv_instance;
int transid = ((Param3*)p)->transid;
int dir = ((Param3*)p)->dir;
string src = ((Param3*)p)->src;
string dst = ((Param3*)p)->dst;
string master_ip = ((Param3*)p)->master_ip;
int master_port = ((Param3*)p)->master_port;
delete (Param3*)p;
if (src.c_str()[0] == '\0')
src = "/" + src;
if (dst.c_str()[0] == '\0')
dst = "/" + dst;
bool success = true;
queue<string> tr; // files to be replicated
queue<string> td; // directories to be explored
if (dir > 0)
td.push(src);
else
tr.push(src);
while (!td.empty())
{
// If the file to be replicated is a directory, recursively list all files first
string src_path = td.front();
td.pop();
// try list this path
SectorMsg msg;
msg.setType(101);
msg.setKey(0);
msg.setData(0, src_path.c_str(), src_path.length() + 1);
Address addr;
self->m_Routing.lookup(src_path, addr);
if (self->m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0)
{
success = false;
break;
}
// the master only returns positive if this is a directory
if (msg.getType() >= 0)
{
// if this is a directory, create it, and put all files and sub-directories into the queue of files to be copied
// create a local dir
string dst_path = dst;
if (src != src_path)
dst_path += "/" + src_path.substr(src.length() + 1, src_path.length() - src.length() - 1);
//create at .tmp first, then move to real location
self->createDir(string(".tmp") + dst_path);
string filelist = msg.getData();
unsigned int s = 0;
while (s < filelist.length())
{
int t = filelist.find(';', s);
SNode sn;
sn.deserialize(filelist.substr(s, t - s).c_str());
if (sn.m_bIsDir)
td.push(src_path + "/" + sn.m_strName);
else
tr.push(src_path + "/" + sn.m_strName);
s = t + 1;
}
continue;
}
}
while (!tr.empty())
{
string src_path = tr.front();
tr.pop();
SNode tmp;
if (self->m_pLocalFile->lookup(src_path.c_str(), tmp) >= 0)
{
//if file is local, copy directly
//note that in this case, src != dst, therefore this is a regular "cp" command, not a system replication
//IMPORTANT!!!
//local files must be read directly from local disk, and cannot be read via datachn due to its limitation
string dst_path = dst;
if (src != src_path)
dst_path += "/" + src_path.substr(src.length() + 1, src_path.length() - src.length() - 1);
//copy to .tmp first, then move to real location
self->createDir(string(".tmp") + dst_path.substr(0, dst_path.rfind('/')));
LocalFS::copy(self->m_strHomeDir + src_path, self->m_strHomeDir + ".tmp" + dst_path);
}
else
{
// open the file and copy it to local
SectorMsg msg;
msg.setType(110);
msg.setKey(0);
int32_t mode = SF_MODE::READ;
msg.setData(0, (char*)&mode, 4);
int32_t localport = self->m_DataChn.getPort();
msg.setData(4, (char*)&localport, 4);
int32_t len_name = src_path.length() + 1;
msg.setData(8, (char*)&len_name, 4);
msg.setData(12, src_path.c_str(), len_name);
int32_t len_opt = 0;
msg.setData(12 + len_name, (char*)&len_opt, 4);
Address addr;
self->m_Routing.lookup(src_path, addr);
if ((self->m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0) || (msg.getType() < 0))
{
success = false;
break;
}
int32_t session = *(int32_t*)msg.getData();
int64_t size = *(int64_t*)(msg.getData() + 4);
time_t ts = *(int64_t*)(msg.getData() + 12);
string ip = msg.getData() + 24;
int32_t port = *(int32_t*)(msg.getData() + 64 + 24);
if (!self->m_DataChn.isConnected(ip, port))
{
if (self->m_DataChn.connect(ip, port) < 0)
{
success = false;
break;
}
}
// download command: 3
int32_t cmd = 3;
self->m_DataChn.send(ip, port, session, (char*)&cmd, 4);
int64_t offset = 0;
self->m_DataChn.send(ip, port, session, (char*)&offset, 8);
int response = -1;
if ((self->m_DataChn.recv4(ip, port, session, response) < 0) || (-1 == response))
{
success = false;
break;
}
string dst_path = dst;
if (src != src_path)
dst_path += "/" + src_path.substr(src.length() + 1, src_path.length() - src.length() - 1);
//copy to .tmp first, then move to real location
self->createDir(string(".tmp") + dst_path.substr(0, dst_path.rfind('/')));
fstream ofs;
ofs.open((self->m_strHomeDir + ".tmp" + dst_path).c_str(), ios::out | ios::binary | ios::trunc);
int64_t unit = 64000000; //send 64MB each time
int64_t torecv = size;
int64_t recd = 0;
while (torecv > 0)
{
int64_t block = (torecv < unit) ? torecv : unit;
if (self->m_DataChn.recvfile(ip, port, session, ofs, offset + recd, block) < 0)
{
success = false;
break;
}
recd += block;
torecv -= block;
}
ofs.close();
// update total received data size
self->m_SlaveStat.updateIO(ip, size, +SlaveStat::SYS_IN);
cmd = 5;
self->m_DataChn.send(ip, port, session, (char*)&cmd, 4);
self->m_DataChn.recv4(ip, port, session, cmd);
if (src == dst)
{
//utime: update timestamp according to the original copy, for replica only; files created by "cp" have new timestamp
utimbuf ut;
ut.actime = ts;
ut.modtime = ts;
utime((self->m_strHomeDir + ".tmp" + dst_path).c_str(), &ut);
}
}
}
if (success)
{
// move from temporary dir to the real dir when the copy is completed
self->createDir(dst.substr(0, dst.rfind('/')));
LocalFS::rename(self->m_strHomeDir + ".tmp" + dst, self->m_strHomeDir + dst);
// if the file has been modified during the replication, remove this replica
int32_t type = (src == dst) ? +FileChangeType::FILE_UPDATE_REPLICA : +FileChangeType::FILE_UPDATE_NEW;
if (self->report(master_ip, master_port, transid, dst, type) < 0)
LocalFS::erase(self->m_strHomeDir + dst);
}
else
{
// failed, remove all temporary files
LocalFS::erase(self->m_strHomeDir + ".tmp" + dst);
self->report(master_ip, master_port, transid, "", +FileChangeType::FILE_UPDATE_NO);
}
// clear this transaction
self->m_TransManager.updateSlave(transid, self->m_iSlaveID);
return NULL;
}
SSDBServer::SSDBServer(SSDB *ssdb, SSDB *meta, const Config &conf, NetworkServer *net){
this->ssdb = (SSDBImpl *)ssdb;
this->meta = meta;
net->data = this;
this->reg_procs(net);
int sync_speed = conf.get_num("replication.sync_speed");
backend_dump = new BackendDump(this->ssdb);
backend_sync = new BackendSync(this->ssdb, sync_speed);
expiration = new ExpirationHandler(this->ssdb);
cluster = new Cluster(this->ssdb);
if(cluster->init() == -1){
log_fatal("cluster init failed!");
exit(1);
}
{ // slaves
const Config *repl_conf = conf.get("replication");
if(repl_conf != NULL){
std::vector<Config *> children = repl_conf->children;
for(std::vector<Config *>::iterator it = children.begin(); it != children.end(); it++){
Config *c = *it;
if(c->key != "slaveof"){
continue;
}
std::string ip = c->get_str("ip");
int port = c->get_num("port");
if(ip == ""){
ip = c->get_str("host");
}
if(ip == "" || port <= 0 || port > 65535){
continue;
}
bool is_mirror = false;
std::string type = c->get_str("type");
if(type == "mirror"){
is_mirror = true;
}else{
type = "sync";
is_mirror = false;
}
std::string id = c->get_str("id");
log_info("slaveof: %s:%d, type: %s", ip.c_str(), port, type.c_str());
Slave *slave = new Slave(ssdb, meta, ip.c_str(), port, is_mirror);
if(!id.empty()){
slave->set_id(id);
}
slave->auth = c->get_str("auth");
slave->start();
slaves.push_back(slave);
}
}
}
// load kv_range
int ret = this->get_kv_range(&this->kv_range_s, &this->kv_range_e);
if(ret == -1){
log_fatal("load key_range failed!");
exit(1);
}
log_info("key_range.kv: \"%s\", \"%s\"",
str_escape(this->kv_range_s).c_str(),
str_escape(this->kv_range_e).c_str()
);
}
