/*
* Attach TCP protocol to socket, allocating internet protocol control
* block, tcp control block, bufer space, and entering LISTEN state
* if to accept connections.
*/
static int
tcp_attach(struct socket *so, struct pru_attach_info *ai)
{
struct tcpcb *tp;
struct inpcb *inp;
int error;
int cpu;
#ifdef INET6
int isipv6 = INP_CHECK_SOCKAF(so, AF_INET6) != 0;
#endif
if (so->so_snd.ssb_hiwat == 0 || so->so_rcv.ssb_hiwat == 0) {
lwkt_gettoken(&so->so_rcv.ssb_token);
error = soreserve(so, tcp_sendspace, tcp_recvspace,
ai->sb_rlimit);
lwkt_reltoken(&so->so_rcv.ssb_token);
if (error)
return (error);
}
atomic_set_int(&so->so_rcv.ssb_flags, SSB_AUTOSIZE);
atomic_set_int(&so->so_snd.ssb_flags, SSB_AUTOSIZE);
cpu = mycpu->gd_cpuid;
/*
* Set the default port for protocol processing. This will likely
* change when we connect.
*/
error = in_pcballoc(so, &tcbinfo[cpu]);
if (error)
return (error);
inp = so->so_pcb;
#ifdef INET6
if (isipv6) {
inp->inp_vflag |= INP_IPV6;
inp->in6p_hops = -1; /* use kernel default */
}
else
#endif
inp->inp_vflag |= INP_IPV4;
tp = tcp_newtcpcb(inp);
if (tp == NULL) {
/*
* Make sure the socket is destroyed by the pcbdetach.
*/
soreference(so);
#ifdef INET6
if (isipv6)
in6_pcbdetach(inp);
else
#endif
in_pcbdetach(inp);
sofree(so); /* from ref above */
return (ENOBUFS);
}
tp->t_state = TCPS_CLOSED;
return (0);
}
/*
* lwkt_thread_replyport() - Backend to lwkt_replymsg()
*
* Called with the reply port as an argument but in the context of the
* original target port. Completion must occur on the target port's
* cpu.
*
* The critical section protects us from IPIs on the this CPU.
*/
static
void
lwkt_thread_replyport(lwkt_port_t port, lwkt_msg_t msg)
{
int flags;
KKASSERT((msg->ms_flags & (MSGF_DONE|MSGF_QUEUED|MSGF_INTRANSIT)) == 0);
if (msg->ms_flags & MSGF_SYNC) {
/*
* If a synchronous completion has been requested, just wakeup
* the message without bothering to queue it to the target port.
*
* Assume the target thread is non-preemptive, so no critical
* section is required.
*/
if (port->mpu_td->td_gd == mycpu) {
crit_enter();
flags = msg->ms_flags;
cpu_sfence();
msg->ms_flags |= MSGF_DONE | MSGF_REPLY;
if (port->mp_flags & MSGPORTF_WAITING)
_lwkt_schedule_msg(port->mpu_td, flags);
crit_exit();
} else {
#ifdef INVARIANTS
atomic_set_int(&msg->ms_flags, MSGF_INTRANSIT);
#endif
atomic_set_int(&msg->ms_flags, MSGF_REPLY);
lwkt_send_ipiq(port->mpu_td->td_gd,
(ipifunc1_t)lwkt_thread_replyport_remote, msg);
}
} else {
/*
* If an asynchronous completion has been requested the message
* must be queued to the reply port.
*
* A critical section is required to interlock the port queue.
*/
if (port->mpu_td->td_gd == mycpu) {
crit_enter();
_lwkt_enqueue_reply(port, msg);
if (port->mp_flags & MSGPORTF_WAITING)
_lwkt_schedule_msg(port->mpu_td, msg->ms_flags);
crit_exit();
} else {
#ifdef INVARIANTS
atomic_set_int(&msg->ms_flags, MSGF_INTRANSIT);
#endif
atomic_set_int(&msg->ms_flags, MSGF_REPLY);
lwkt_send_ipiq(port->mpu_td->td_gd,
(ipifunc1_t)lwkt_thread_replyport_remote, msg);
}
}
}
/*
* Kill all lwps associated with the current process except the
* current lwp. Return an error if we race another thread trying to
* do the same thing and lose the race.
*
* If forexec is non-zero the current thread and process flags are
* cleaned up so they can be reused.
*
* Caller must hold curproc->p_token
*/
int
killalllwps(int forexec)
{
struct lwp *lp = curthread->td_lwp;
struct proc *p = lp->lwp_proc;
int fakestop;
/*
* Interlock against P_WEXIT. Only one of the process's thread
* is allowed to do the master exit.
*/
if (p->p_flags & P_WEXIT)
return (EALREADY);
p->p_flags |= P_WEXIT;
/*
* Set temporary stopped state in case we are racing a coredump.
* Otherwise the coredump may hang forever.
*/
if (lp->lwp_mpflags & LWP_MP_WSTOP) {
fakestop = 0;
} else {
atomic_set_int(&lp->lwp_mpflags, LWP_MP_WSTOP);
++p->p_nstopped;
fakestop = 1;
wakeup(&p->p_nstopped);
}
/*
* Interlock with LWP_MP_WEXIT and kill any remaining LWPs
*/
atomic_set_int(&lp->lwp_mpflags, LWP_MP_WEXIT);
if (p->p_nthreads > 1)
killlwps(lp);
/*
* Undo temporary stopped state
*/
if (fakestop) {
atomic_clear_int(&lp->lwp_mpflags, LWP_MP_WSTOP);
--p->p_nstopped;
}
/*
* If doing this for an exec, clean up the remaining thread
* (us) for continuing operation after all the other threads
* have been killed.
*/
if (forexec) {
atomic_clear_int(&lp->lwp_mpflags, LWP_MP_WEXIT);
p->p_flags &= ~P_WEXIT;
}
return(0);
}
static int
ng_ksocket_connect(hook_p hook)
{
node_p node = NG_HOOK_NODE(hook);
const priv_p priv = NG_NODE_PRIVATE(node);
struct socket *const so = priv->so;
/* Add our hook for incoming data and other events */
priv->so->so_upcallarg = (caddr_t)node;
priv->so->so_upcall = ng_ksocket_incoming;
atomic_set_int(&priv->so->so_rcv.ssb_flags, SSB_UPCALL);
atomic_set_int(&priv->so->so_snd.ssb_flags, SSB_UPCALL);
/*
* --Original comment--
* On a cloned socket we may have already received one or more
* upcalls which we couldn't handle without a hook. Handle
* those now.
* We cannot call the upcall function directly
* from here, because until this function has returned our
* hook isn't connected.
*
* ---meta comment for -current ---
* XXX This is dubius.
* Upcalls between the time that the hook was
* first created and now (on another processesor) will
* be earlier on the queue than the request to finalise the hook.
* By the time the hook is finalised,
* The queued upcalls will have happenned and the code
* will have discarded them because of a lack of a hook.
* (socket not open).
*
* This is a bad byproduct of the complicated way in which hooks
* are now created (3 daisy chained async events).
*
* Since we are a netgraph operation
* We know that we hold a lock on this node. This forces the
* request we make below to be queued rather than implemented
* immediatly which will cause the upcall function to be called a bit
* later.
* However, as we will run any waiting queued operations immediatly
* after doing this one, if we have not finalised the other end
* of the hook, those queued operations will fail.
*/
if (priv->flags & KSF_CLONED) {
ng_send_fn(node, NULL, &ng_ksocket_incoming2, so, M_WAITOK | M_NULLOK);
}
return (0);
}
/*
* Set only flag to suggest that device is suspended. This call is
* not supported in NetBSD.
*
*/
int
dm_dev_suspend_ioctl(prop_dictionary_t dm_dict)
{
dm_dev_t *dmv;
const char *name, *uuid;
uint32_t flags, minor;
name = NULL;
uuid = NULL;
flags = 0;
prop_dictionary_get_cstring_nocopy(dm_dict, DM_IOCTL_NAME, &name);
prop_dictionary_get_cstring_nocopy(dm_dict, DM_IOCTL_UUID, &uuid);
prop_dictionary_get_uint32(dm_dict, DM_IOCTL_FLAGS, &flags);
prop_dictionary_get_uint32(dm_dict, DM_IOCTL_MINOR, &minor);
if ((dmv = dm_dev_lookup(name, uuid, minor)) == NULL) {
DM_REMOVE_FLAG(flags, DM_EXISTS_FLAG);
return ENOENT;
}
atomic_set_int(&dmv->flags, DM_SUSPEND_FLAG);
dm_dbg_print_flags(dmv->flags);
prop_dictionary_set_uint32(dm_dict, DM_IOCTL_OPEN, dmv->table_head.io_cnt);
prop_dictionary_set_uint32(dm_dict, DM_IOCTL_FLAGS, dmv->flags);
prop_dictionary_set_uint32(dm_dict, DM_IOCTL_MINOR, dmv->minor);
dm_dev_unbusy(dmv);
/* Add flags to dictionary flag after dmv -> dict copy */
DM_ADD_FLAG(flags, DM_EXISTS_FLAG);
return 0;
}
void
fbsdrun_addcpu(struct vmctx *ctx, int vcpu, uint64_t rip)
{
int error;
if (cpumask & (1 << vcpu)) {
fprintf(stderr, "addcpu: attempting to add existing cpu %d\n",
vcpu);
exit(1);
}
atomic_set_int(&cpumask, 1 << vcpu);
/*
* Set up the vmexit struct to allow execution to start
* at the given RIP
*/
vmexit[vcpu].rip = rip;
vmexit[vcpu].inst_length = 0;
mt_vmm_info[vcpu].mt_ctx = ctx;
mt_vmm_info[vcpu].mt_vcpu = vcpu;
error = pthread_create(&mt_vmm_info[vcpu].mt_thr, NULL,
fbsdrun_start_thread, &mt_vmm_info[vcpu]);
assert(error == 0);
}
/*
* Create the OS-specific port helper thread and per-port lock.
*/
void
ahci_os_start_port(struct ahci_port *ap)
{
char name[16];
atomic_set_int(&ap->ap_signal, AP_SIGF_INIT | AP_SIGF_THREAD_SYNC);
lockinit(&ap->ap_lock, "ahcipo", 0, 0);
lockinit(&ap->ap_sim_lock, "ahcicam", 0, LK_CANRECURSE);
lockinit(&ap->ap_sig_lock, "ahport", 0, 0);
sysctl_ctx_init(&ap->sysctl_ctx);
ksnprintf(name, sizeof(name), "%d", ap->ap_num);
ap->sysctl_tree = SYSCTL_ADD_NODE(&ap->sysctl_ctx,
SYSCTL_CHILDREN(ap->ap_sc->sysctl_tree),
OID_AUTO, name, CTLFLAG_RD, 0, "");
if ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SALP) &&
(ap->ap_sc->sc_cap & (AHCI_REG_CAP_PSC | AHCI_REG_CAP_SSC))) {
SYSCTL_ADD_PROC(&ap->sysctl_ctx,
SYSCTL_CHILDREN(ap->sysctl_tree), OID_AUTO,
"link_pwr_mgmt", CTLTYPE_INT | CTLFLAG_RW, ap, 0,
ahci_sysctl_link_pwr_mgmt, "I",
"Link power management policy "
"(0 = disabled, 1 = medium, 2 = aggressive)");
SYSCTL_ADD_PROC(&ap->sysctl_ctx,
SYSCTL_CHILDREN(ap->sysctl_tree), OID_AUTO,
"link_pwr_state", CTLTYPE_STRING | CTLFLAG_RD, ap, 0,
ahci_sysctl_link_pwr_state, "A",
"Link power management state");
}
kthread_create(ahci_port_thread, ap, &ap->ap_thread,
"%s", PORTNAME(ap));
}
static
int
lwkt_thread_putport(lwkt_port_t port, lwkt_msg_t msg)
{
KKASSERT((msg->ms_flags & (MSGF_DONE | MSGF_REPLY)) == 0);
msg->ms_target_port = port;
if (port->mpu_td->td_gd == mycpu) {
crit_enter();
_lwkt_pushmsg(port, msg);
if (port->mp_flags & MSGPORTF_WAITING)
_lwkt_schedule_msg(port->mpu_td, msg->ms_flags);
crit_exit();
} else {
#ifdef INVARIANTS
/*
* Cleanup.
*
* An atomic op is needed on ms_flags vs originator. Also
* note that the originator might be using a different type
* of msgport.
*/
atomic_set_int(&msg->ms_flags, MSGF_INTRANSIT);
#endif
lwkt_send_ipiq(port->mpu_td->td_gd,
(ipifunc1_t)lwkt_thread_putport_remote, msg);
}
return (EASYNC);
}
int
_pthread_setcanceltype(int type, int *oldtype)
{
struct pthread *curthread = tls_get_curthread();
int oldval;
oldval = curthread->cancelflags;
if (oldtype != NULL)
*oldtype = ((oldval & THR_CANCEL_AT_POINT) ?
PTHREAD_CANCEL_ASYNCHRONOUS :
PTHREAD_CANCEL_DEFERRED);
switch (type) {
case PTHREAD_CANCEL_ASYNCHRONOUS:
atomic_set_int(&curthread->cancelflags, THR_CANCEL_AT_POINT);
testcancel(curthread);
break;
case PTHREAD_CANCEL_DEFERRED:
atomic_clear_int(&curthread->cancelflags, THR_CANCEL_AT_POINT);
break;
default:
return (EINVAL);
}
return (0);
}
/*
* Handle a predicate event request. This function is only called once
* when the predicate message queueing request is received.
*/
void
netmsg_so_notify(netmsg_t msg)
{
struct lwkt_token *tok;
struct signalsockbuf *ssb;
ssb = (msg->notify.nm_etype & NM_REVENT) ?
&msg->base.nm_so->so_rcv :
&msg->base.nm_so->so_snd;
/*
* Reply immediately if the event has occured, otherwise queue the
* request.
*
* NOTE: Socket can change if this is an accept predicate so cache
* the token.
*/
tok = lwkt_token_pool_lookup(msg->base.nm_so);
lwkt_gettoken(tok);
if (msg->notify.nm_predicate(&msg->notify)) {
lwkt_reltoken(tok);
lwkt_replymsg(&msg->base.lmsg,
msg->base.lmsg.ms_error);
} else {
TAILQ_INSERT_TAIL(&ssb->ssb_kq.ki_mlist, &msg->notify, nm_list);
atomic_set_int(&ssb->ssb_flags, SSB_MEVENT);
lwkt_reltoken(tok);
}
}
// we assume to have exclusive ownership of DATA: we're the only ones allowed
// to write to it as the owning thread is (normally) dying.
// This assumption should remain valid until the moment we put the node (data)
// in the place of the "free" field of the SMR global data, at which time
// we relinquish ownership.
// A thread dequeueing a node from free should not assume to have ownership until
// it succeeds a CAS setting the new value of the free field in the SMR global data.
// That's why we set that field to NULL to assume ownership of the node as well.
// Normally, no-one should be traversing the queue of nodes at that time, but
// we take some measures to accomodate for that (im)possibility anyway.
// Also note we've added a flag to the local data type (hptr_local_data_t)
// which is 1 if the hptr scanning logic should no longer count on our
// "next" field to be valid.
static void smr_hptr_free_list_enq(hptr_local_data_t * data)
{
unsigned int i;
hptr_local_data_t * exp;
hptr_local_data_t * ptr = NULL;
// make our node ready for inclusion in the free list
// clear out all existing hazardous references
for (i = 0; i < smr_global_data->k; i++)
atomic_set_ptr(&(data->hp[i]), NULL);
// set out flag
atomic_set_int(&(data->flag), 1);
// NOTE: we expect atomic_set_* to do the right thing concerning
// memory barriers: we expect the setting of the flag to be visible
// before the clearing of the next pointer!
// clear out our next pointer
atomic_set_ptr(&(data->next), NULL);
// try to set it to smr_global_data->free first.
exp = NULL;
while (compare_and_exchange_ptr(&exp, &(smr_global_data->free), data) != 0)
{ // Otherwise, set smr_global_data->free to NULL (and get whatever is in there);
// set its old value to our next and put us in its place.
// "Whatever is in there" is what we have in exp now..
if (compare_and_exchange_ptr(&exp, &(smr_global_data->free), NULL) != 0)
continue;
// we may have been here more than once, in which case ptr holds whatever we have in our next field..
if (compare_and_exchange_ptr(&ptr, &(data->next), NULL) != 0)
assert(0); // just in case..
// we now put whatever is in ptr at the end of exp - which might be a whole queue for all we know..
if (exp != NULL && ptr != NULL)
{
hptr_local_data_t * head = exp;
hptr_local_data_t * tail = exp;
while (1)
{
// find the tail of the queue in exp (now head)
while (tail->next != NULL)
tail = tail->next;
// we expect a NULL at the end of this queue
exp = NULL;
// try to put ptr at the end of the queue
if (compare_and_exchange_ptr(&exp, &(tail->next), ptr) != 0)
{ // otherwise, we start over from scratch
tail = head;
continue;
}
}
// when we get here, ptr is at the end of the queue we want to add to our node
// so we add the queue...
exp = NULL;
if (compare_and_exchange_ptr(&exp, &(data->next), head) != 0)
// someone was still writing to our node - that should be impossible..
assert(0);
}
exp = NULL;
}
// when we get here, we're done.
}
/*
* Kill all lwps associated with the current process except the
* current lwp. Return an error if we race another thread trying to
* do the same thing and lose the race.
*
* If forexec is non-zero the current thread and process flags are
* cleaned up so they can be reused.
*
* Caller must hold curproc->p_token
*/
int
killalllwps(int forexec)
{
struct lwp *lp = curthread->td_lwp;
struct proc *p = lp->lwp_proc;
/*
* Interlock against P_WEXIT. Only one of the process's thread
* is allowed to do the master exit.
*/
if (p->p_flags & P_WEXIT)
return (EALREADY);
p->p_flags |= P_WEXIT;
/*
* Interlock with LWP_MP_WEXIT and kill any remaining LWPs
*/
atomic_set_int(&lp->lwp_mpflags, LWP_MP_WEXIT);
if (p->p_nthreads > 1)
killlwps(lp);
/*
* If doing this for an exec, clean up the remaining thread
* (us) for continuing operation after all the other threads
* have been killed.
*/
if (forexec) {
atomic_clear_int(&lp->lwp_mpflags, LWP_MP_WEXIT);
p->p_flags &= ~P_WEXIT;
}
return(0);
}
/*
* Add (mask) to the set of bits being sent to the per-port thread helper
* and wake the helper up if necessary.
*/
void
sili_os_signal_port_thread(struct sili_port *ap, int mask)
{
lockmgr(&ap->ap_sig_lock, LK_EXCLUSIVE);
atomic_set_int(&ap->ap_signal, mask);
wakeup(&ap->ap_thread);
lockmgr(&ap->ap_sig_lock, LK_RELEASE);
}
/*
* Release a VX lock that also held a ref on the vnode. vrele() will handle
* any needed state transitions.
*
* However, filesystems use this function to get rid of unwanted new vnodes
* so try to get the vnode on the correct queue in that case.
*/
void
vx_put(struct vnode *vp)
{
if (vp->v_type == VNON || vp->v_type == VBAD)
atomic_set_int(&vp->v_refcnt, VREF_FINALIZE);
lockmgr(&vp->v_lock, LK_RELEASE);
vrele(vp);
}
/*
* Notify host that there are data pending on our TX bufring.
*/
static __inline void
vmbus_chan_signal_tx(const struct vmbus_channel *chan)
{
atomic_set_long(chan->ch_evtflag, chan->ch_evtflag_mask);
if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF)
atomic_set_int(chan->ch_montrig, chan->ch_montrig_mask);
else
hypercall_signal_event(chan->ch_monprm_dma.hv_paddr);
}
/*
* Mark an inode as being modified, meaning that the caller will modify
* ip->meta.
*
* If a vnode is present we set the vnode dirty and the nominal filesystem
* sync will also handle synchronizing the inode meta-data. If no vnode
* is present we must ensure that the inode is on pmp->sideq.
*
* NOTE: We must always queue the inode to the sideq. This allows H2 to
* shortcut vsyncscan() and flush inodes and their related vnodes
* in a two stages. H2 still calls vfsync() for each vnode.
*
* NOTE: No mtid (modify_tid) is passed into this routine. The caller is
* only modifying the in-memory inode. A modify_tid is synchronized
* later when the inode gets flushed.
*
* NOTE: As an exception to the general rule, the inode MAY be locked
* shared for this particular call.
*/
void
hammer2_inode_modify(hammer2_inode_t *ip)
{
atomic_set_int(&ip->flags, HAMMER2_INODE_MODIFIED);
if (ip->vp)
vsetisdirty(ip->vp);
if (ip->pmp && (ip->flags & HAMMER2_INODE_NOSIDEQ) == 0)
hammer2_inode_delayed_sideq(ip);
}
inline void ATOMIC_SET(ATOMIC_T *v, int i)
{
#ifdef PLATFORM_LINUX
atomic_set(v,i);
#elif defined(PLATFORM_WINDOWS)
*v=i;// other choice????
#elif defined(PLATFORM_FREEBSD)
atomic_set_int(v,i);
#endif
}
/*
* Create the OS-specific port helper thread and per-port lock.
*/
void
sili_os_start_port(struct sili_port *ap)
{
atomic_set_int(&ap->ap_signal, AP_SIGF_INIT);
lockinit(&ap->ap_lock, "silipo", 0, LK_CANRECURSE);
lockinit(&ap->ap_sim_lock, "silicam", 0, LK_CANRECURSE);
lockinit(&ap->ap_sig_lock, "siport", 0, 0);
kthread_create(sili_port_thread, ap, &ap->ap_thread,
"%s", PORTNAME(ap));
}
/*
* Asynchronous remaster request. Ask the synchronization thread to
* start over soon (as if it were frozen and unfrozen, but without waiting).
* The thread always recalculates mastership relationships when restarting.
*/
void
hammer2_thr_remaster(hammer2_thread_t *thr)
{
if (thr->td == NULL)
return;
lockmgr(&thr->lk, LK_EXCLUSIVE);
atomic_set_int(&thr->flags, HAMMER2_THREAD_REMASTER);
wakeup(&thr->flags);
lockmgr(&thr->lk, LK_RELEASE);
}
static __inline
void
_lwkt_enqueue_reply(lwkt_port_t port, lwkt_msg_t msg)
{
/*
* atomic op needed for spin ports
*/
_lwkt_pushmsg(port, msg);
atomic_set_int(&msg->ms_flags, MSGF_REPLY | MSGF_DONE);
}
int
copyout_nofault(const void *kaddr, void *udaddr, size_t len)
{
thread_t td = curthread;
int error;
atomic_set_int(&td->td_flags, TDF_NOFAULT);
error = copyout(kaddr, udaddr, len);
atomic_clear_int(&td->td_flags, TDF_NOFAULT);
return error;
}
/*
* uiomove() but fail for non-trivial VM faults, even if the VM fault is
* valid. Returns EFAULT if a VM fault occurred via the copyin/copyout
* onfault code.
*
* This allows callers to hold e.g. a busy VM page, or a busy VM object,
* or a locked vnode through the call and then fall-back to safer code
* if we fail.
*/
int
uiomove_nofault(caddr_t cp, size_t n, struct uio *uio)
{
thread_t td = curthread;
int error;
atomic_set_int(&td->td_flags, TDF_NOFAULT);
error = uiomove(cp, n, uio);
atomic_clear_int(&td->td_flags, TDF_NOFAULT);
return error;
}
/*
* Obtain a new vnode. The returned vnode is VX locked & vrefd.
*
* All new vnodes set the VAGE flags. An open() of the vnode will
* decrement the (2-bit) flags. Vnodes which are opened several times
* are thus retained in the cache over vnodes which are merely stat()d.
*
* We always allocate the vnode. Attempting to recycle existing vnodes
* here can lead to numerous deadlocks, particularly with softupdates.
*/
struct vnode *
allocvnode(int lktimeout, int lkflags)
{
struct vnode *vp;
/*
* Do not flag for synchronous recyclement unless there are enough
* freeable vnodes to recycle and the number of vnodes has
* significantly exceeded our target. We want the normal vnlru
* process to handle the cleaning (at 9/10's) before we are forced
* to flag it here at 11/10's for userexit path processing.
*/
if (numvnodes >= maxvnodes * 11 / 10 &&
cachedvnodes + inactivevnodes >= maxvnodes * 5 / 10) {
struct thread *td = curthread;
if (td->td_lwp)
atomic_set_int(&td->td_lwp->lwp_mpflags, LWP_MP_VNLRU);
}
/*
* lktimeout only applies when LK_TIMELOCK is used, and only
* the pageout daemon uses it. The timeout may not be zero
* or the pageout daemon can deadlock in low-VM situations.
*/
if (lktimeout == 0)
lktimeout = hz / 10;
vp = kmalloc(sizeof(*vp), M_VNODE, M_ZERO | M_WAITOK);
lwkt_token_init(&vp->v_token, "vnode");
lockinit(&vp->v_lock, "vnode", lktimeout, lkflags);
TAILQ_INIT(&vp->v_namecache);
RB_INIT(&vp->v_rbclean_tree);
RB_INIT(&vp->v_rbdirty_tree);
RB_INIT(&vp->v_rbhash_tree);
spin_init(&vp->v_spin, "allocvnode");
lockmgr(&vp->v_lock, LK_EXCLUSIVE);
atomic_add_int(&numvnodes, 1);
vp->v_refcnt = 1;
vp->v_flag = VAGE0 | VAGE1;
vp->v_pbuf_count = nswbuf_kva / NSWBUF_SPLIT;
KKASSERT(TAILQ_EMPTY(&vp->v_namecache));
/* exclusive lock still held */
vp->v_filesize = NOOFFSET;
vp->v_type = VNON;
vp->v_tag = 0;
vp->v_state = VS_CACHED;
_vactivate(vp);
return (vp);
}
int
_thr_cancel_enter(struct pthread *curthread)
{
int oldval;
oldval = curthread->cancelflags;
if (!(oldval & THR_CANCEL_AT_POINT)) {
atomic_set_int(&curthread->cancelflags, THR_CANCEL_AT_POINT);
testcancel(curthread);
}
return (oldval);
}
void
hammer2_cluster_set_chainflags(hammer2_cluster_t *cluster, uint32_t flags)
{
hammer2_chain_t *chain;
int i;
for (i = 0; i < cluster->nchains; ++i) {
chain = cluster->array[i];
if (chain)
atomic_set_int(&chain->flags, flags);
}
}
/*
* Set an inode's cluster modified, marking the related chains RW and
* duplicating them if necessary.
*
* The passed-in chain is a localized copy of the chain previously acquired
* when the inode was locked (and possilby replaced in the mean time), and
* must also be updated. In fact, we update it first and then synchronize
* the inode's cluster cache.
*/
hammer2_inode_data_t *
hammer2_cluster_modify_ip(hammer2_trans_t *trans, hammer2_inode_t *ip,
hammer2_cluster_t *cluster, int flags)
{
atomic_set_int(&ip->flags, HAMMER2_INODE_MODIFIED);
hammer2_cluster_modify(trans, cluster, flags);
hammer2_inode_repoint(ip, NULL, cluster);
if (ip->vp)
vsetisdirty(ip->vp);
return (&hammer2_cluster_wdata(cluster)->ipdata);
}
/*
* Handle a predicate event request. This function is only called once
* when the predicate message queueing request is received.
*/
void
netmsg_so_notify(netmsg_t msg)
{
struct lwkt_token *tok;
struct signalsockbuf *ssb;
ssb = (msg->notify.nm_etype & NM_REVENT) ?
&msg->base.nm_so->so_rcv :
&msg->base.nm_so->so_snd;
/*
* Reply immediately if the event has occured, otherwise queue the
* request.
*
* NOTE: Socket can change if this is an accept predicate so cache
* the token.
*/
tok = lwkt_token_pool_lookup(msg->base.nm_so);
lwkt_gettoken(tok);
atomic_set_int(&ssb->ssb_flags, SSB_MEVENT);
if (msg->notify.nm_predicate(&msg->notify)) {
if (TAILQ_EMPTY(&ssb->ssb_kq.ki_mlist))
atomic_clear_int(&ssb->ssb_flags, SSB_MEVENT);
lwkt_reltoken(tok);
lwkt_replymsg(&msg->base.lmsg,
msg->base.lmsg.ms_error);
} else {
TAILQ_INSERT_TAIL(&ssb->ssb_kq.ki_mlist, &msg->notify, nm_list);
/*
* NOTE:
* If predict ever blocks, 'tok' will be released, so
* SSB_MEVENT set beforehand could have been cleared
* when we reach here. In case that happens, we set
* SSB_MEVENT again, after the notify has been queued.
*/
atomic_set_int(&ssb->ssb_flags, SSB_MEVENT);
lwkt_reltoken(tok);
}
}
void
hammer2_thr_freeze(hammer2_thread_t *thr)
{
if (thr->td == NULL)
return;
lockmgr(&thr->lk, LK_EXCLUSIVE);
atomic_set_int(&thr->flags, HAMMER2_THREAD_FREEZE);
wakeup(&thr->flags);
while ((thr->flags & HAMMER2_THREAD_FROZEN) == 0) {
lksleep(thr, &thr->lk, 0, "h2frz", hz);
}
lockmgr(&thr->lk, LK_RELEASE);
}
/*
* Start a XOP request, queueing it to all nodes in the cluster to
* execute the cluster op.
*
* XXX optimize single-target case.
*/
void
hammer2_xop_start_except(hammer2_xop_head_t *xop, hammer2_xop_func_t func,
int notidx)
{
hammer2_xop_group_t *xgrp;
hammer2_thread_t *thr;
hammer2_pfs_t *pmp;
int g;
int i;
pmp = xop->ip->pmp;
if (pmp->has_xop_threads == 0)
hammer2_xop_helper_create(pmp);
g = pmp->xop_iterator++;
g = g & HAMMER2_XOPGROUPS_MASK;
xgrp = &pmp->xop_groups[g];
xop->func = func;
xop->xgrp = xgrp;
/* XXX do cluster_resolve or cluster_check here, only start
* synchronized elements */
for (i = 0; i < xop->ip->cluster.nchains; ++i) {
thr = &xgrp->thrs[i];
if (thr->td && i != notidx) {
lockmgr(&thr->lk, LK_EXCLUSIVE);
if (thr->td &&
(thr->flags & HAMMER2_THREAD_STOP) == 0) {
atomic_set_int(&xop->run_mask, 1U << i);
atomic_set_int(&xop->chk_mask, 1U << i);
TAILQ_INSERT_TAIL(&thr->xopq, xop,
collect[i].entry);
}
lockmgr(&thr->lk, LK_RELEASE);
wakeup(&thr->flags);
}
}
}
/*
* Terminate a thread. This function will silently return if the thread
* was never initialized or has already been deleted.
*
* This is accomplished by setting the STOP flag and waiting for the td
* structure to become NULL.
*/
void
hammer2_thr_delete(hammer2_thread_t *thr)
{
if (thr->td == NULL)
return;
lockmgr(&thr->lk, LK_EXCLUSIVE);
atomic_set_int(&thr->flags, HAMMER2_THREAD_STOP);
wakeup(&thr->flags);
while (thr->td) {
lksleep(thr, &thr->lk, 0, "h2thr", hz);
}
lockmgr(&thr->lk, LK_RELEASE);
thr->pmp = NULL;
lockuninit(&thr->lk);
}
