void xpc_disconnect_wait(int ch_number) { unsigned long irq_flags; short partid; struct xpc_partition *part; struct xpc_channel *ch; int wakeup_channel_mgr; /* now wait for all callouts to the caller's function to cease */ for (partid = 1; partid < XP_MAX_PARTITIONS; partid++) { part = &xpc_partitions[partid]; if (!xpc_part_ref(part)) continue; ch = &part->channels[ch_number]; if (!(ch->flags & XPC_C_WDISCONNECT)) { xpc_part_deref(part); continue; } wait_for_completion(&ch->wdisconnect_wait); spin_lock_irqsave(&ch->lock, irq_flags); DBUG_ON(!(ch->flags & XPC_C_DISCONNECTED)); wakeup_channel_mgr = 0; if (ch->delayed_IPI_flags) { if (part->act_state != XPC_P_DEACTIVATING) { spin_lock(&part->IPI_lock); XPC_SET_IPI_FLAGS(part->local_IPI_amo, ch->number, ch->delayed_IPI_flags); spin_unlock(&part->IPI_lock); wakeup_channel_mgr = 1; } ch->delayed_IPI_flags = 0; } ch->flags &= ~XPC_C_WDISCONNECT; spin_unlock_irqrestore(&ch->lock, irq_flags); if (wakeup_channel_mgr) xpc_wakeup_channel_mgr(part); xpc_part_deref(part); } }
void xpc_disconnect_wait(int ch_number) { unsigned long irq_flags; short partid; struct xpc_partition *part; struct xpc_channel *ch; int wakeup_channel_mgr; /* */ for (partid = 0; partid < xp_max_npartitions; partid++) { part = &xpc_partitions[partid]; if (!xpc_part_ref(part)) continue; ch = &part->channels[ch_number]; if (!(ch->flags & XPC_C_WDISCONNECT)) { xpc_part_deref(part); continue; } wait_for_completion(&ch->wdisconnect_wait); spin_lock_irqsave(&ch->lock, irq_flags); DBUG_ON(!(ch->flags & XPC_C_DISCONNECTED)); wakeup_channel_mgr = 0; if (ch->delayed_chctl_flags) { if (part->act_state != XPC_P_AS_DEACTIVATING) { spin_lock(&part->chctl_lock); part->chctl.flags[ch->number] |= ch->delayed_chctl_flags; spin_unlock(&part->chctl_lock); wakeup_channel_mgr = 1; } ch->delayed_chctl_flags = 0; } ch->flags &= ~XPC_C_WDISCONNECT; spin_unlock_irqrestore(&ch->lock, irq_flags); if (wakeup_channel_mgr) xpc_wakeup_channel_mgr(part); xpc_part_deref(part); } }
/* * XPC's heartbeat code calls this function to inform XPC that a partition is * going down. XPC responds by tearing down the XPartition Communication * infrastructure used for the just downed partition. * * XPC's heartbeat code will never call this function and xpc_partition_up() * at the same time. Nor will it ever make multiple calls to either function * at the same time. */ void xpc_partition_going_down(struct xpc_partition *part, enum xp_retval reason) { unsigned long irq_flags; int ch_number; struct xpc_channel *ch; dev_dbg(xpc_chan, "deactivating partition %d, reason=%d\n", XPC_PARTID(part), reason); if (!xpc_part_ref(part)) { /* infrastructure for this partition isn't currently set up */ return; } /* disconnect channels associated with the partition going down */ for (ch_number = 0; ch_number < part->nchannels; ch_number++) { ch = &part->channels[ch_number]; xpc_msgqueue_ref(ch); spin_lock_irqsave(&ch->lock, irq_flags); XPC_DISCONNECT_CHANNEL(ch, reason, &irq_flags); spin_unlock_irqrestore(&ch->lock, irq_flags); xpc_msgqueue_deref(ch); } xpc_wakeup_channel_mgr(part); xpc_part_deref(part); }
static irqreturn_t xpc_handle_notify_IRQ_uv(int irq, void *dev_id) { struct xpc_notify_mq_msg_uv *msg; short partid; struct xpc_partition *part; while ((msg = gru_get_next_message(xpc_notify_mq_uv->gru_mq_desc)) != NULL) { partid = msg->hdr.partid; if (partid < 0 || partid >= XP_MAX_NPARTITIONS_UV) { dev_err(xpc_part, "xpc_handle_notify_IRQ_uv() received " "invalid partid=0x%x in message\n", partid); } else { part = &xpc_partitions[partid]; if (xpc_part_ref(part)) { xpc_handle_notify_mq_msg_uv(part, msg); xpc_part_deref(part); } } gru_free_message(xpc_notify_mq_uv->gru_mq_desc, msg); } return IRQ_HANDLED; }
void xpc_partition_going_down(struct xpc_partition *part, enum xp_retval reason) { unsigned long irq_flags; int ch_number; struct xpc_channel *ch; dev_dbg(xpc_chan, "deactivating partition %d, reason=%d\n", XPC_PARTID(part), reason); if (!xpc_part_ref(part)) { return; } for (ch_number = 0; ch_number < part->nchannels; ch_number++) { ch = &part->channels[ch_number]; xpc_msgqueue_ref(ch); spin_lock_irqsave(&ch->lock, irq_flags); XPC_DISCONNECT_CHANNEL(ch, reason, &irq_flags); spin_unlock_irqrestore(&ch->lock, irq_flags); xpc_msgqueue_deref(ch); } xpc_wakeup_channel_mgr(part); xpc_part_deref(part); }
/* * When XPC HB determines that a partition has come up, it will create a new * kthread and that kthread will call this function to attempt to set up the * basic infrastructure used for Cross Partition Communication with the newly * upped partition. * * The kthread that was created by XPC HB and which setup the XPC * infrastructure will remain assigned to the partition until the partition * goes down. At which time the kthread will teardown the XPC infrastructure * and then exit. * * XPC HB will put the remote partition's XPC per partition specific variables * physical address into xpc_partitions[partid].remote_vars_part_pa prior to * calling xpc_partition_up(). */ static void xpc_partition_up(struct xpc_partition *part) { DBUG_ON(part->channels != NULL); dev_dbg(xpc_chan, "activating partition %d\n", XPC_PARTID(part)); if (xpc_setup_infrastructure(part) != xpSuccess) return; /* * The kthread that XPC HB called us with will become the * channel manager for this partition. It will not return * back to XPC HB until the partition's XPC infrastructure * has been dismantled. */ (void)xpc_part_ref(part); /* this will always succeed */ if (xpc_make_first_contact(part) == xpSuccess) xpc_channel_mgr(part); xpc_part_deref(part); xpc_teardown_infrastructure(part); }
/* * Check to see if xpc_notify_IRQ_handler() dropped any IPIs on the floor * because the write to their associated IPI amo completed after the IRQ/IPI * was received. */ void xpc_dropped_IPI_check(struct xpc_partition *part) { if (xpc_part_ref(part)) { xpc_check_for_channel_activity(part); part->dropped_IPI_timer.expires = jiffies + XPC_P_DROPPED_IPI_WAIT; add_timer(&part->dropped_IPI_timer); xpc_part_deref(part); } }
/* * When XPC HB determines that a partition has come up, it will create a new * kthread and that kthread will call this function to attempt to set up the * basic infrastructure used for Cross Partition Communication with the newly * upped partition. * * The kthread that was created by XPC HB and which setup the XPC * infrastructure will remain assigned to the partition becoming the channel * manager for that partition until the partition is deactivating, at which * time the kthread will teardown the XPC infrastructure and then exit. */ static int xpc_activating(void *__partid) { short partid = (u64)__partid; struct xpc_partition *part = &xpc_partitions[partid]; unsigned long irq_flags; DBUG_ON(partid < 0 || partid >= xp_max_npartitions); spin_lock_irqsave(&part->act_lock, irq_flags); if (part->act_state == XPC_P_AS_DEACTIVATING) { part->act_state = XPC_P_AS_INACTIVE; spin_unlock_irqrestore(&part->act_lock, irq_flags); part->remote_rp_pa = 0; return 0; } /* indicate the thread is activating */ DBUG_ON(part->act_state != XPC_P_AS_ACTIVATION_REQ); part->act_state = XPC_P_AS_ACTIVATING; XPC_SET_REASON(part, 0, 0); spin_unlock_irqrestore(&part->act_lock, irq_flags); dev_dbg(xpc_part, "activating partition %d\n", partid); xpc_allow_hb(partid); if (xpc_setup_ch_structures(part) == xpSuccess) { (void)xpc_part_ref(part); /* this will always succeed */ if (xpc_make_first_contact(part) == xpSuccess) { xpc_mark_partition_active(part); xpc_channel_mgr(part); /* won't return until partition is deactivating */ } xpc_part_deref(part); xpc_teardown_ch_structures(part); } xpc_disallow_hb(partid); xpc_mark_partition_inactive(part); if (part->reason == xpReactivating) { /* interrupting ourselves results in activating partition */ xpc_request_partition_reactivation(part); } return 0; }
/* * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more * than one partition, we use an AMO_t structure per partition to indicate * whether a partition has sent an IPI or not. If it has, then wake up the * associated kthread to handle it. * * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IPIs sent by XPC * running on other partitions. * * Noteworthy Arguments: * * irq - Interrupt ReQuest number. NOT USED. * * dev_id - partid of IPI's potential sender. */ irqreturn_t xpc_notify_IRQ_handler(int irq, void *dev_id) { short partid = (short)(u64)dev_id; struct xpc_partition *part = &xpc_partitions[partid]; DBUG_ON(partid <= 0 || partid >= XP_MAX_PARTITIONS); if (xpc_part_ref(part)) { xpc_check_for_channel_activity(part); xpc_part_deref(part); } return IRQ_HANDLED; }
void xpc_initiate_connect(int ch_number) { short partid; struct xpc_partition *part; struct xpc_channel *ch; DBUG_ON(ch_number < 0 || ch_number >= XPC_MAX_NCHANNELS); for (partid = 0; partid < xp_max_npartitions; partid++) { part = &xpc_partitions[partid]; if (xpc_part_ref(part)) { ch = &part->channels[ch_number]; xpc_wakeup_channel_mgr(part); xpc_part_deref(part); } } }
/* * Send a message that contains the user's payload on the specified channel * connected to the specified partition. * * NOTE that this routine can sleep waiting for a message entry to become * available. To not sleep, pass in the XPC_NOWAIT flag. * * Once sent, this routine will not wait for the message to be received, nor * will notification be given when it does happen. * * Arguments: * * partid - ID of partition to which the channel is connected. * ch_number - channel # to send message on. * flags - see xp.h for valid flags. * payload - pointer to the payload which is to be sent. * payload_size - size of the payload in bytes. */ enum xp_retval xpc_initiate_send(short partid, int ch_number, u32 flags, void *payload, u16 payload_size) { struct xpc_partition *part = &xpc_partitions[partid]; enum xp_retval ret = xpUnknownReason; dev_dbg(xpc_chan, "payload=0x%p, partid=%d, channel=%d\n", payload, partid, ch_number); DBUG_ON(partid < 0 || partid >= xp_max_npartitions); DBUG_ON(ch_number < 0 || ch_number >= part->nchannels); DBUG_ON(payload == NULL); if (xpc_part_ref(part)) { ret = xpc_send_payload(&part->channels[ch_number], flags, payload, payload_size, 0, NULL, NULL); xpc_part_deref(part); } return ret; }
static irqreturn_t xpc_handle_activate_IRQ_uv(int irq, void *dev_id) { struct xpc_activate_mq_msghdr_uv *msg_hdr; short partid; struct xpc_partition *part; int wakeup_hb_checker = 0; int part_referenced; while (1) { msg_hdr = gru_get_next_message(xpc_activate_mq_uv->gru_mq_desc); if (msg_hdr == NULL) break; partid = msg_hdr->partid; if (partid < 0 || partid >= XP_MAX_NPARTITIONS_UV) { dev_err(xpc_part, "xpc_handle_activate_IRQ_uv() " "received invalid partid=0x%x in message\n", partid); } else { part = &xpc_partitions[partid]; part_referenced = xpc_part_ref(part); xpc_handle_activate_mq_msg_uv(part, msg_hdr, part_referenced, &wakeup_hb_checker); if (part_referenced) xpc_part_deref(part); } gru_free_message(xpc_activate_mq_uv->gru_mq_desc, msg_hdr); } if (wakeup_hb_checker) wake_up_interruptible(&xpc_activate_IRQ_wq); return IRQ_HANDLED; }
/* * Called by XP at the time of channel connection unregistration to cause * XPC to teardown all current connections for the specified channel. * * Before returning xpc_initiate_disconnect() will wait until all connections * on the specified channel have been closed/torndown. So the caller can be * assured that they will not be receiving any more callouts from XPC to the * function they registered via xpc_connect(). * * Arguments: * * ch_number - channel # to unregister. */ void xpc_initiate_disconnect(int ch_number) { unsigned long irq_flags; short partid; struct xpc_partition *part; struct xpc_channel *ch; DBUG_ON(ch_number < 0 || ch_number >= XPC_MAX_NCHANNELS); /* initiate the channel disconnect for every active partition */ for (partid = 0; partid < xp_max_npartitions; partid++) { part = &xpc_partitions[partid]; if (xpc_part_ref(part)) { ch = &part->channels[ch_number]; xpc_msgqueue_ref(ch); spin_lock_irqsave(&ch->lock, irq_flags); if (!(ch->flags & XPC_C_DISCONNECTED)) { ch->flags |= XPC_C_WDISCONNECT; XPC_DISCONNECT_CHANNEL(ch, xpUnregistering, &irq_flags); } spin_unlock_irqrestore(&ch->lock, irq_flags); xpc_msgqueue_deref(ch); xpc_part_deref(part); } } xpc_disconnect_wait(ch_number); }
/* * Called by XP at the time of channel connection registration to cause * XPC to establish connections to all currently active partitions. */ void xpc_initiate_connect(int ch_number) { short partid; struct xpc_partition *part; struct xpc_channel *ch; DBUG_ON(ch_number < 0 || ch_number >= XPC_MAX_NCHANNELS); for (partid = 0; partid < xp_max_npartitions; partid++) { part = &xpc_partitions[partid]; if (xpc_part_ref(part)) { ch = &part->channels[ch_number]; /* * Initiate the establishment of a connection on the * newly registered channel to the remote partition. */ xpc_wakeup_channel_mgr(part); xpc_part_deref(part); } } }
void xpc_create_kthreads(struct xpc_channel *ch, int needed, int ignore_disconnecting) { unsigned long irq_flags; u64 args = XPC_PACK_ARGS(ch->partid, ch->number); struct xpc_partition *part = &xpc_partitions[ch->partid]; struct task_struct *kthread; void (*indicate_partition_disengaged) (struct xpc_partition *) = xpc_arch_ops.indicate_partition_disengaged; while (needed-- > 0) { /* */ if (ignore_disconnecting) { if (!atomic_inc_not_zero(&ch->kthreads_assigned)) { /* */ BUG_ON(!(ch->flags & XPC_C_DISCONNECTINGCALLOUT_MADE)); break; } } else if (ch->flags & XPC_C_DISCONNECTING) { break; } else if (atomic_inc_return(&ch->kthreads_assigned) == 1 && atomic_inc_return(&part->nchannels_engaged) == 1) { xpc_arch_ops.indicate_partition_engaged(part); } (void)xpc_part_ref(part); xpc_msgqueue_ref(ch); kthread = kthread_run(xpc_kthread_start, (void *)args, "xpc%02dc%d", ch->partid, ch->number); if (IS_ERR(kthread)) { /* */ /* */ if (atomic_dec_return(&ch->kthreads_assigned) == 0 && atomic_dec_return(&part->nchannels_engaged) == 0) { indicate_partition_disengaged(part); } xpc_msgqueue_deref(ch); xpc_part_deref(part); if (atomic_read(&ch->kthreads_assigned) < ch->kthreads_idle_limit) { /* */ spin_lock_irqsave(&ch->lock, irq_flags); XPC_DISCONNECT_CHANNEL(ch, xpLackOfResources, &irq_flags); spin_unlock_irqrestore(&ch->lock, irq_flags); } break; } } }
/* * For each partition that XPC has established communications with, there is * a minimum of one kernel thread assigned to perform any operation that * may potentially sleep or block (basically the callouts to the asynchronous * functions registered via xpc_connect()). * * Additional kthreads are created and destroyed by XPC as the workload * demands. * * A kthread is assigned to one of the active channels that exists for a given * partition. */ void xpc_create_kthreads(struct xpc_channel *ch, int needed, int ignore_disconnecting) { unsigned long irq_flags; u64 args = XPC_PACK_ARGS(ch->partid, ch->number); struct xpc_partition *part = &xpc_partitions[ch->partid]; struct task_struct *kthread; while (needed-- > 0) { /* * The following is done on behalf of the newly created * kthread. That kthread is responsible for doing the * counterpart to the following before it exits. */ if (ignore_disconnecting) { if (!atomic_inc_not_zero(&ch->kthreads_assigned)) { /* kthreads assigned had gone to zero */ BUG_ON(!(ch->flags & XPC_C_DISCONNECTINGCALLOUT_MADE)); break; } } else if (ch->flags & XPC_C_DISCONNECTING) { break; } else if (atomic_inc_return(&ch->kthreads_assigned) == 1 && atomic_inc_return(&part->nchannels_engaged) == 1) { xpc_indicate_partition_engaged(part); } (void)xpc_part_ref(part); xpc_msgqueue_ref(ch); kthread = kthread_run(xpc_kthread_start, (void *)args, "xpc%02dc%d", ch->partid, ch->number); if (IS_ERR(kthread)) { /* the fork failed */ /* * NOTE: if (ignore_disconnecting && * !(ch->flags & XPC_C_DISCONNECTINGCALLOUT)) is true, * then we'll deadlock if all other kthreads assigned * to this channel are blocked in the channel's * registerer, because the only thing that will unblock * them is the xpDisconnecting callout that this * failed kthread_run() would have made. */ if (atomic_dec_return(&ch->kthreads_assigned) == 0 && atomic_dec_return(&part->nchannels_engaged) == 0) { xpc_indicate_partition_disengaged(part); } xpc_msgqueue_deref(ch); xpc_part_deref(part); if (atomic_read(&ch->kthreads_assigned) < ch->kthreads_idle_limit) { /* * Flag this as an error only if we have an * insufficient #of kthreads for the channel * to function. */ spin_lock_irqsave(&ch->lock, irq_flags); XPC_DISCONNECT_CHANNEL(ch, xpLackOfResources, &irq_flags); spin_unlock_irqrestore(&ch->lock, irq_flags); } break; } } }
static int xpc_kthread_start(void *args) { short partid = XPC_UNPACK_ARG1(args); u16 ch_number = XPC_UNPACK_ARG2(args); struct xpc_partition *part = &xpc_partitions[partid]; struct xpc_channel *ch; int n_needed; unsigned long irq_flags; dev_dbg(xpc_chan, "kthread starting, partid=%d, channel=%d\n", partid, ch_number); ch = &part->channels[ch_number]; if (!(ch->flags & XPC_C_DISCONNECTING)) { /* let registerer know that connection has been established */ spin_lock_irqsave(&ch->lock, irq_flags); if (!(ch->flags & XPC_C_CONNECTEDCALLOUT)) { ch->flags |= XPC_C_CONNECTEDCALLOUT; spin_unlock_irqrestore(&ch->lock, irq_flags); xpc_connected_callout(ch); spin_lock_irqsave(&ch->lock, irq_flags); ch->flags |= XPC_C_CONNECTEDCALLOUT_MADE; spin_unlock_irqrestore(&ch->lock, irq_flags); /* * It is possible that while the callout was being * made that the remote partition sent some messages. * If that is the case, we may need to activate * additional kthreads to help deliver them. We only * need one less than total #of messages to deliver. */ n_needed = xpc_n_of_deliverable_payloads(ch) - 1; if (n_needed > 0 && !(ch->flags & XPC_C_DISCONNECTING)) xpc_activate_kthreads(ch, n_needed); } else { spin_unlock_irqrestore(&ch->lock, irq_flags); } xpc_kthread_waitmsgs(part, ch); } /* let registerer know that connection is disconnecting */ spin_lock_irqsave(&ch->lock, irq_flags); if ((ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) && !(ch->flags & XPC_C_DISCONNECTINGCALLOUT)) { ch->flags |= XPC_C_DISCONNECTINGCALLOUT; spin_unlock_irqrestore(&ch->lock, irq_flags); xpc_disconnect_callout(ch, xpDisconnecting); spin_lock_irqsave(&ch->lock, irq_flags); ch->flags |= XPC_C_DISCONNECTINGCALLOUT_MADE; } spin_unlock_irqrestore(&ch->lock, irq_flags); if (atomic_dec_return(&ch->kthreads_assigned) == 0 && atomic_dec_return(&part->nchannels_engaged) == 0) { xpc_indicate_partition_disengaged(part); } xpc_msgqueue_deref(ch); dev_dbg(xpc_chan, "kthread exiting, partid=%d, channel=%d\n", partid, ch_number); xpc_part_deref(part); return 0; }
static int xpc_kthread_start(void *args) { short partid = XPC_UNPACK_ARG1(args); u16 ch_number = XPC_UNPACK_ARG2(args); struct xpc_partition *part = &xpc_partitions[partid]; struct xpc_channel *ch; int n_needed; unsigned long irq_flags; int (*n_of_deliverable_payloads) (struct xpc_channel *) = xpc_arch_ops.n_of_deliverable_payloads; dev_dbg(xpc_chan, "kthread starting, partid=%d, channel=%d\n", partid, ch_number); ch = &part->channels[ch_number]; if (!(ch->flags & XPC_C_DISCONNECTING)) { /* */ spin_lock_irqsave(&ch->lock, irq_flags); if (!(ch->flags & XPC_C_CONNECTEDCALLOUT)) { ch->flags |= XPC_C_CONNECTEDCALLOUT; spin_unlock_irqrestore(&ch->lock, irq_flags); xpc_connected_callout(ch); spin_lock_irqsave(&ch->lock, irq_flags); ch->flags |= XPC_C_CONNECTEDCALLOUT_MADE; spin_unlock_irqrestore(&ch->lock, irq_flags); /* */ n_needed = n_of_deliverable_payloads(ch) - 1; if (n_needed > 0 && !(ch->flags & XPC_C_DISCONNECTING)) xpc_activate_kthreads(ch, n_needed); } else { spin_unlock_irqrestore(&ch->lock, irq_flags); } xpc_kthread_waitmsgs(part, ch); } /* */ spin_lock_irqsave(&ch->lock, irq_flags); if ((ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) && !(ch->flags & XPC_C_DISCONNECTINGCALLOUT)) { ch->flags |= XPC_C_DISCONNECTINGCALLOUT; spin_unlock_irqrestore(&ch->lock, irq_flags); xpc_disconnect_callout(ch, xpDisconnecting); spin_lock_irqsave(&ch->lock, irq_flags); ch->flags |= XPC_C_DISCONNECTINGCALLOUT_MADE; } spin_unlock_irqrestore(&ch->lock, irq_flags); if (atomic_dec_return(&ch->kthreads_assigned) == 0 && atomic_dec_return(&part->nchannels_engaged) == 0) { xpc_arch_ops.indicate_partition_disengaged(part); } xpc_msgqueue_deref(ch); dev_dbg(xpc_chan, "kthread exiting, partid=%d, channel=%d\n", partid, ch_number); xpc_part_deref(part); return 0; }