void
pmaint::pmaint_lookup (bigint key, dhash_stat err, vec<chord_node> sl, route r)
{
if (err) {
warning << host_node->my_ID () << " lookup failed. key " << key << ", err " << err << "\n";
pmaint_next (); //XXX delay?
return;
}
assert (r.size () >= 2);
assert (sl.size () >= 1);
chordID succ = r.pop_back ()->id ();
chordID pred = r.pop_back ()->id ();
assert (succ == sl[0].x);
if (dhblock_chash::num_efrags () > sl.size ()) {
warning << "not enough successors: " << sl.size ()
<< " vs " << dhblock_chash::num_efrags () << "\n";
//try again later
active_cb = delaycb (PRTTMLONG, wrap (this, &pmaint::pmaint_next));
return;
}
if ((sl.size () > dhblock_chash::num_efrags() &&
dhblock_chash::num_efrags () > 1 &&
betweenbothincl (sl[0].x, sl[sl.size () - 1].x,
host_node->my_ID ()))
||
(dhblock_chash::num_efrags () == 1 && sl[0].x == host_node->my_ID ())
//above is a special case since between always returns true
// when the first two arguments are equal
) {
trace << host_node->my_ID () << " PMAINT: we are a replica for "
<< key << " " << sl[0].x << " -- " << sl[dhblock_chash::num_efrags () - 1]
<< "\n";
//case I: we are a replica of the key.
//i.e. in the successor list. Do nothing.
//next time we'll do a lookup with the next key
active_cb = delaycb (PRTTMTINY, wrap (this, &pmaint::pmaint_next));
} else {
//case II: this key doesn't belong to us. Offer it to another node
pmaint_searching = false;
trace << host_node->my_ID () << ": offering " << key << "\n";
pmaint_offer (key, sl[0]);
}
}
void
stabilize_manager::stabilize_continuous (u_int32_t t)
{
stabilize_continuous_tmo = NULL;
if (continuous_stabilizing ()) { // stabilizing too fast
t = 2 * t;
warnx << gettime () << " " << myID
<<" stabilize_continuous: slow down " << t << "\n";
} else {
// Try to keep t around cts_timer_base
if (t >= cts_timer_base) t -= stabilize_decrease_timer;
if (t < cts_timer_base/2) t = (int)(stabilize_slowdown_factor * t);
for (unsigned int i = 0; i < clients.size (); i++)
clients[i]->do_continuous ();
}
if (t > cts_timer_max)
t = cts_timer_max;
u_int32_t t1 = uniform_random (0.5 * t, 1.5 * t);
u_int32_t sec = t1 / 1000;
u_int32_t nsec = (t1 % 1000) * 1000000;
continuous_timer = t;
stabilize_continuous_tmo =
delaycb (sec, nsec, wrap (this,
&stabilize_manager::stabilize_continuous, t));
}
void
filesrv::fhetimeout (void)
{
purgefhe ();
purgefd (0);
fhetmo = delaycb (fhe_timer, wrap (this, &filesrv::fhetimeout));
}
void
hclient_t::connected (ptr<bool> d_local, int f)
{
if (*d_local) {
if (f > 0)
::close (f);
stats.timeouts ++;
warn << "timed out before connect succeeded\n";
return;
}
fd = f;
if (fd < 0) {
stats.rejections ++;
warn << "connection rejected\n";
cexit (-1);
return;
}
int d = get_cli_write_delay ();
if (d == 0)
sched_write (destroyed);
else {
wcb = delaycb (d / THOUSAND, (d % THOUSAND) * MILLION,
wrap (this, &hclient_t::sched_write, destroyed));
}
}
static void
restart ()
{
if (hup_lock)
return;
hup_lock = true;
warn ("version %s, pid %d, restarted with SIGHUP\n", VERSION,
int (getpid ()));
server *s, *ns;
for (s = serverlist.first; s; s = ns) {
ns = serverlist.next (s);
serverlist.remove (s);
delete s;
}
for (sfssrv *nsp, *sp = services.first; sp; sp = nsp) {
nsp = services.next (sp);
delete sp;
}
for (sockaddr **sap = listenaddrs.base (); sap < listenaddrs.lim (); sap++)
xfree (*sap);
listenaddrs.clear ();
listeners.clear ();
parseconfig ();
delaycb (0, 100000000, wrap (launchservers));
}
void
sfsserver_auth::keygen ()
{
privkey = sfscrypt.gen (SFS_RABIN, 0, SFS_DECRYPT);
if (!keytmo)
keytmo = delaycb (3536 + (rnd.getword () & 0x3f), 0, wrap (&keyexpire));
}
noise_from_prog::noise_from_prog (datasink *dst, const char *const *av, cbv cb)
: cb (cb)
{
int fd = execprog (av);
to = delaycb (30, wrap (this, &noise_from_prog::timeout));
getfdnoise (dst, fd, wrap (this, &noise_from_prog::done));
}
void
vnode_impl::check_dead_node_cb (ptr<location> l, time_t nbackoff, chordstat s)
{
unsigned int i=0;
chordID id = l->id ();
for (i=0; i<dead_nodes.size (); i++)
if (dead_nodes[i]->id () == id)
break;
if (s != CHORD_OK) {
if (i == dead_nodes.size ())
dead_nodes.push_back (l);
delaycb (nbackoff, wrap (this, &vnode_impl::check_dead_node, l, nbackoff));
}
else {
/* Take it off the dead list */
if (i != dead_nodes.size ()) {
dead_nodes[i] = dead_nodes[0];
dead_nodes.pop_front ();
}
// Insertion should ensure that node is set to alive and in table
// (Liveness should also have been set by ping_cb)
ptr<location> nl = locations->insert (l);
if (nl != l) {
warn << "duplicate location " << l << "\n";
}
notify (my_succ (), myID);
}
}
void
http_parser_base_t::parse (cbi c)
{
cb = c;
tocb = delaycb (timeout, 0, wrap (this, &http_parser_base_t::clnt_timeout));
_parsing_header = true;
hdr_p ()->parse (wrap (this, &http_parser_base_t::parse_cb1));
}
void
pmaint::start ()
{
if (active_cb)
return;
int jitter = uniform_random (0, PRTTMLONG);
active_cb = delaycb (PRTTMLONG + jitter, wrap (this, &pmaint::pmaint_next));
}
void
vnode_impl::doRPC_cb (ptr<location> l, xdrproc_t proc,
void *out, aclnt_cb cb,
ref<dorpc_res> res, clnt_stat err)
{
if (err) {
ptr<location> reall = locations->lookup (l->id ());
if (reall && reall->alive ()) {
warn << "got error " << err << ", but " << l
<< " is still marked alive\n";
reall->set_alive (false);
} else if (!reall) {
locations->insert (l);
l->set_alive (false);
}
if (!l->alive () && checkdead_int > 0) {
// benjie: no longer alive, put it on the dead_nodes list so
// we can try to contact it periodically
unsigned i=0;
for (i=0; i<dead_nodes.size (); i++)
if (dead_nodes[i]->id () == l->id ())
break;
if (i == dead_nodes.size ()) {
dead_nodes.push_back (l);
delaycb (checkdead_int, wrap (this, &vnode_impl::check_dead_node, l, checkdead_int));
}
}
cb (err);
}
else if (res->status != DORPC_OK)
cb (RPC_CANTRECV);
else {
float distance = l->distance ();
chord_node n = make_chord_node (res->resok->src);
l->set_coords (n);
Coord u_coords (n);
update_coords (u_coords, distance);
// This should reset the age of the node to zero because
// remote side always provides an age of zero for self
// and locationtable will pull in updates that are younger.
if (me_->id () != n.x)
locations->insert (n);
//unmarshall the result and copy to out
xdrmem x ((char *)res->resok->results.base (),
res->resok->results.size (), XDR_DECODE);
assert (proc);
if (!proc (x.xdrp (), out)) {
fatal << "failed to unmarshall result\n";
cb (RPC_CANTSEND);
} else
cb (err);
}
}
long
vnode_impl::doRPC (ref<location> l, const rpc_program &prog, int procno,
ptr<void> in, void *out, aclnt_cb cb,
cbtmo_t cb_tmo /* = NULL */, bool stream /* = false */)
{
//check to see if this is alive
ptr<location> loc = locations->lookup (l->id ());
if (loc && !loc->alive()) {
warn << my_ID () << ": doRPC (" << prog.name << "." << procno
<< ") on dead node " << l->address () << "\n";
delaycb (0, wrap (&err_cb, cb));
return -1;
}
ptr<dorpc_arg> arg = marshal_doRPC (l, prog, procno, in);
if (!arg) {
cb (RPC_CANTSEND);
return 0;
} else {
// This is the real call; cf transport tracking in stp_manager.
track_call (prog, procno, arg->args.size ());
ref<dorpc_res> res = New refcounted<dorpc_res> (DORPC_OK);
xdrproc_t outproc = prog.tbl[procno].xdr_res;
u_int32_t xid = random_getword ();
// Stolen (mostly) from aclnt::init_call
if (aclnttrace >= 2) {
str name;
const rpcgen_table *rtp;
rtp = &prog.tbl[procno];
assert (rtp);
name = strbuf ("%s:%s fake_xid=%x", prog.name, rtp->name, xid)
<< " on " << l->address () << ":" << l->vnode ();
warn << "ACLNT_TRACE:" << tracetime () << " call " << name << "\n";
if (aclnttrace >= 5 && rtp->print_arg)
rtp->print_arg (in, NULL, aclnttrace - 4, "ARGS", "");
if (aclnttrace >= 3 && cb != aclnt_cb_null)
cb = wrap (printreply, cb, name, out, rtp->print_res);
}
aclnt_cb cbw = wrap (this, &vnode_impl::doRPC_cb,
l, outproc, out, cb, res);
if (!stream)
return rpcm->doRPC (me_, l, transport_program_1, TRANSPORTPROC_DORPC,
arg, res, cbw,
wrap(this, &vnode_impl::tmo, cb_tmo,
prog.progno, procno, arg->args.size ()));
else
return rpcm->doRPC_stream (me_, l,
transport_program_1, TRANSPORTPROC_DORPC,
arg, res, cbw);
}
}
void
dhblock_srv::start (bool randomize)
{
int delay = 0;
if (!repair_tcb) {
if (randomize)
delay = random_getword () % dhash::reptm ();
repair_tcb = delaycb (dhash::reptm () + delay,
wrap (this, &dhblock_srv::repair_timer));
}
}
void ping_start() {
if (self_.mid == view_.primary.mid) {
ping_incoming_.clear();
ping_timeout_cb_ =
delaycb(periodic_primary_ping + timeout_primary_ping,
wrap(mkref(this), &cohort::ping_check_incoming));
} else {
ping_timeout_cb_ =
delaycb(periodic_primary_ping,
wrap(mkref(this), &cohort::ping_start));
}
ping_arg arg;
arg.myid = self_.mid;
arg.vid = view_.vid;
sockaddr_in sin = netaddr2sockaddr(view_.primary.addr);
ref<ping_res> resp = New refcounted<ping_res>();
c_->timedcall(timeout_primary_ping, REP_PING, &arg, resp,
wrap(mkref(this), &cohort::ping_cb, view_.vid, resp),
0, 0, 0, 0, 0, (sockaddr *) &sin);
}
static void
phase2cb (timespec mintime, int numtogo)
{
timespec ts;
clock_gettime (CLOCK_REALTIME, &ts);
if (mintime > ts)
panic ("callback too early\n");
if (!--numtogo)
exit (0);
ts.tv_sec++;
delaycb (1, wrap (phase2cb, ts, numtogo));
}
void
pmaint::pmaint_offer_cb (chord_node dst, bigint key,
ref<dhash_offer_res> res,
clnt_stat err)
{
if (err) {
warning << host_node->my_ID () << " error offering key " << key << ", retrying.\n";
vec<adb_keyaux_t> keys;
adb_keyaux_t dummy;
dummy.key = key;
delaycb (PRTTMSHORT, wrap (this, &pmaint::pmaint_gotkey, ADB_OK, pmaint_getkeys_id, keys));
return;
} else {
switch (res->resok->accepted[0]) {
case DHASH_DELETE:
case DHASH_HOLD:
{
//do nothing for now. delete if disk space is a problem
trace << host_node->my_ID () << ": holding " << key << "\n";
}
break;
case DHASH_SENDTO:
{
chord_node dst = make_chord_node (res->resok->dest[0]);
trace << host_node->my_ID () << ": sending " << key << " to " << dst.x << "\n";
pmaint_handoff (dst, key, wrap (this, &pmaint::handed_off_cb, key));
return;
}
break;
default:
fatal << "unkown dhash_offer type\n";
}
}
pmaint_searching = true;
active_cb = delaycb (PRTTMSHORT, wrap (this, &pmaint::pmaint_next));
return;
}
void
closure_t::end_of_scope_checks (int line)
{
if (tame_check_leaks ()) {
// Unfortunately, we can only perform these end of scope checks
// from the event loop, since we need to wait for the callstack
// to unwind. Of course, we can't hold a reference to the
// closure since that will keep it from going out of scope.
// So instead, we hold onto the relevant pieces inside the class,
// with an expensive copy in the case of the _rvs.
delaycb (0, 0, wrap (::end_of_scope_checks, _rvs));
}
}
void
dnsreq::fail (int err)
{
assert (err);
if (!error)
error = err;
if (constructed)
readreply (NULL);
else {
remove ();
delaycb (0, wrap (this, &dnsreq::readreply, (dnsparse *) NULL));
}
}
void
tcp_manager::send_RPC (RPC_delay_args *args)
{
hostinfo *hi = lookup_host (args->l->address ());
if (!hi->xp) {
delaycb (0, 0, wrap (this, &tcp_manager::send_RPC_ateofcb, args));
}
else if (hi->xp->ateof()) {
hostlru.remove (hi);
hostlru.insert_tail (hi);
args->l->set_alive (false);
remove_host (hi);
delaycb (0, 0, wrap (this, &tcp_manager::send_RPC_ateofcb, args));
}
else {
hi->orpc++;
args->now = getusec ();
ptr<aclnt> c = aclnt::alloc (hi->xp, args->prog);
c->call (args->procno, args->in, args->out,
wrap (this, &tcp_manager::doRPC_tcp_cleanup, c, args));
}
}
void
proxy::setcb (int i)
{
/* If we have data or an EOF to transmit, schedule a write callback. */
if (con[i].wbuf.resid () || con[!i].eof && !con[i].closed) {
fdcb (con[i].fd, selwrite, wrap (this, &proxy::wcb, i));
if (!con[i].tmo)
con[i].tmo = delaycb (120, wrap (this, &proxy::timeout, i));
}
/* If buffer falls beneath low-water mark, schedule a read callback. */
if (!con[!i].eof && con[i].wbuf.resid () <= lowat)
fdcb (con[!i].fd, selread, wrap (this, &proxy::rcb, !i));
}
void
pmaint::pmaint_gotkey (adb_status stat, u_int32_t id, vec<adb_keyaux_t> keys)
{
if (stat == ADB_OK && keys.size () > 0) {
chordID key = keys[0].key;
pmaint_getkeys_id = id;
vec<ptr<location> > preds = host_node->preds ();
if (preds.size () >= dhblock_chash::num_efrags () &&
betweenrightincl (preds[dhblock_chash::num_efrags () - 1]->id(),
host_node->my_ID (),
key)) {
active_cb = delaycb (PRTTMTINY, wrap (this, &pmaint::pmaint_next));
return;
}
cli->lookup (key, wrap (this, &pmaint::pmaint_lookup, key));
active_cb = NULL;
} else {
//data base is empty
//check back later
info << host_node->my_ID () << " PMAINT NEXT: db empty\n";
active_cb = delaycb (PRTTMLONG, wrap (this, &pmaint::pmaint_next));
}
}
void
dhblock_srv::repair_timer ()
{
if (repairs_queued.size () < REPAIR_QUEUE_MAX)
generate_repair_jobs ();
// Generation will finish asynchronously
// REPAIR_QUEUE_MAX is a soft cap to try and
// keep us from generating too many repairs; subclasses
// are free to call repair_add to enqueue jobs whenever
// they like.
repair_flush_q ();
repair_tcb = delaycb (dhash::reptm (),
wrap (this, &dhblock_srv::repair_timer));
}
static PyObject *
Core_nowcb (PyObject *self, PyObject *args)
{
PyObject *cb;
if (!PyArg_ParseTuple (args, "O", &cb))
return NULL;
if (!PyCallable_Check (cb)) {
PyErr_SetString (PyExc_TypeError, "expected a callable object");
return NULL;
}
delaycb (0, 0, wrap (Core_cb, pop_t::alloc (cb)));
Py_INCREF (Py_None);
return Py_None;
}
void
hclient_t::run ()
{
struct timespec tsnow = sfs_get_tsnow ();
if (noisy) warn << "run: " << id << "\n";
if (nrunning >= nconcur) {
if (noisy) warn << "queuing: " << id << "\n";
q.push_back (this);
return;
}
nrunning++;
if (noisy) warn << "running: " << id << " (nrunning: " << nrunning << ")\n";
if (noisy) warn << "connecting to: " << host << "\n";
cli_start = tsnow;
tcb = delaycb (timeout, 0, wrap (this, &hclient_t::timed_out, destroyed));
tcpconnect (host, port, wrap (this, &hclient_t::connected, destroyed));
}
void
pmaint::handed_off_cb (bigint key, int status)
{
if (status == PMAINT_HANDOFF_ERROR) {
warning << host_node->my_ID () << " error handing off key " << key << "\n";
return;
} else {
strbuf buf;
buf << host_node->my_ID () << " handed off key " << key << " ";
if (status == PMAINT_HANDOFF_NOTPRESENT)
buf << "(was not present).\n";
else
buf << "(was present).\n";
warning << buf;
}
pmaint_searching = true;
active_cb = delaycb (PRTTMSHORT, wrap (this, &pmaint::pmaint_next));
}
static PyObject *
Core_delaycb (PyObject *self, PyObject *args)
{
PyObject *handler;
int s, us;
if (!PyArg_ParseTuple (args, "iiO", &s, &us, &handler))
return NULL;
if (!PyCallable_Check (handler)) {
PyErr_SetString (PyExc_TypeError, "expected a callable object as 3rd arg");
return NULL;
}
delaycb (s, us, wrap (Core_cb, pop_t::alloc (handler)));
Py_INCREF (Py_None);
return Py_None;
}
void
afsusrroot::finish (ref<setupstate> ss, int err)
{
if (!ss->dpt)
return;
if (!err)
err = srvinfo::geterr (ss->name);
if (err) {
ss->dpt->setres (nfsstat (err));
ss->dpt = NULL;
delaycb (15, wrap (afsdir_unlink, mkref (this), ss->name));
return;
}
if (!ss->revdone)
return;
afsnode *e = lookup (ss->name, NULL);
if (!e || e == ss->dpt)
e = afs_sfsroot->lookup (ss->name, NULL);
if (e) {
str name = ss->name;
unlink (name);
link (e, name);
#if FIX_MNTPOINT
if (opt_fix_mntpoint) {
if (str res = e->readlink ())
ss->dpt->setres (res);
else {
warn << "afsusrroot::finish: shouldn't get here (please report bug)\n";
ss->dpt->setres (strbuf ("%s/%s", sfsroot, ss->name.cstr ()));
ss->dpt->setres (strbuf ("%s/" MPDOT "%s/r", sfsroot,
ss->name.cstr ()));
}
}
else
#endif /* FIX_MNTPOINT */
ss->dpt->setres (strbuf ("%s/%s", sfsroot, ss->name.cstr ()));
}
else
srvinfo::alloc (ss->name, wrap (mkref (this), &afsusrroot::finish, ss));
}
void ping_cb(viewid_t vid, ref<ping_res> resp, clnt_stat stat) {
if (view_.vid != vid)
return;
if (stat) {
ping_stop();
view_is_down();
return;
}
if (!resp->ok) {
/*
* This is in case we're fetching logs from the new primary,
* and we're stepping through a number of views quickly.
* Timeout will reset us only if we fail to reach the final
* view within periodic_primary_ping.
*/
ping_stop();
ping_timeout_cb_ =
delaycb(periodic_primary_ping,
wrap(mkref(this), &cohort::view_is_down));
}
}
void
stabilize_manager::stabilize_backoff (u_int32_t t)
{
bool stablenow = isstable ();
stabilize_backoff_tmo = NULL;
if (!stable && stablenow) {
stable = true;
warnx << gettime () << " " << myID
<< ": stabilize: stable! stabilize timer " << t << "\n";
} else if (!stablenow) {
stable = false;
}
if (backoff_stabilizing ()) {
t *= 2;
// warnx << gettime () << " " << myID
// <<" stabilize_backoff: slow down " << t << "\n";
} else {
for (unsigned int i = 0; i < clients.size (); i++)
clients[i]->do_backoff ();
if (stablenow && (t <= bo_timer_max))
t = (int)(stabilize_slowdown_factor * t);
else if (t > bo_timer_base) // ring is unstable; speed up stabilization
t -= stabilize_decrease_timer;
}
if (t > bo_timer_max)
t = bo_timer_max;
u_int32_t t1 = uniform_random (0.5 * t, 1.5 * t);
u_int32_t sec = t1 / 1000;
u_int32_t nsec = (t1 % 1000) * 1000000;
backoff_timer = t;
stabilize_backoff_tmo =
delaycb (sec, nsec, wrap (this, &stabilize_manager::stabilize_backoff, t));
}
void
get_cb (chordID next)
{
draw_ring ();
if (get_queue.size ()) {
chordID n = get_queue.pop_front ();
f_node *nu = nodes[n];
if (nu)
update (nu);
} else {
f_node *node_next = nodes[next];
if (node_next == NULL)
node_next = nodes.first ();
if (node_next) {
update (node_next);
node_next = nodes.next (node_next);
if (node_next == NULL)
node_next = nodes.first ();
next = node_next->ID;
} // else no nodes, keep calling get_cb until there are some
}
delaycb (0, 1000*1000*interval, wrap (&get_cb, next));
}