void
zrl_add_impl(zrlock_t *zrl, const char *zc)
{
for (;;) {
uint32_t n = (uint32_t)zrl->zr_refcount;
while (n != ZRL_LOCKED) {
uint32_t cas = atomic_cas_32(
(uint32_t *)&zrl->zr_refcount, n, n + 1);
if (cas == n) {
ASSERT3S((int32_t)n, >=, 0);
#ifdef ZFS_DEBUG
if (zrl->zr_owner == curthread) {
DTRACE_PROBE2(zrlock__reentry,
zrlock_t *, zrl, uint32_t, n);
}
zrl->zr_owner = curthread;
zrl->zr_caller = zc;
#endif
return;
}
n = cas;
}
mutex_enter(&zrl->zr_mtx);
while (zrl->zr_refcount == ZRL_LOCKED) {
cv_wait(&zrl->zr_cv, &zrl->zr_mtx);
}
mutex_exit(&zrl->zr_mtx);
}
/*
* Allocate an unassigned memnode.
*/
int
mem_node_alloc()
{
int mnode;
mnodeset_t newmask, oldmask;
/*
* Find an unused memnode. Update it atomically to prevent
* a first time memnode creation race.
*/
for (mnode = 0; mnode < max_mem_nodes; mnode++)
if (atomic_cas_32((uint32_t *)&mem_node_config[mnode].exists,
0, 1) == 0)
break;
if (mnode >= max_mem_nodes)
panic("Out of free memnodes\n");
mem_node_config[mnode].physbase = (uint64_t)-1;
mem_node_config[mnode].physmax = 0;
atomic_inc_16(&num_memnodes);
do {
oldmask = memnodes_mask;
newmask = memnodes_mask | (1ull << mnode);
} while (atomic_cas_64(&memnodes_mask, oldmask, newmask) != oldmask);
return (mnode);
}
zrl_add(zrlock_t *zrl)
#endif
{
uint32_t n = (uint32_t)zrl->zr_refcount;
while (n != ZRL_LOCKED) {
uint32_t cas = atomic_cas_32(
(uint32_t *)&zrl->zr_refcount, n, n + 1);
if (cas == n) {
ASSERT((int32_t)n >= 0);
#ifdef ZFS_DEBUG
if (zrl->zr_owner == curthread) {
DTRACE_PROBE2(zrlock__reentry,
zrlock_t *, zrl, uint32_t, n);
}
zrl->zr_owner = curthread;
zrl->zr_caller = zc;
#endif
return;
}
n = cas;
}
mutex_enter(&zrl->zr_mtx);
while (zrl->zr_refcount == ZRL_LOCKED) {
cv_wait(&zrl->zr_cv, &zrl->zr_mtx);
}
ASSERT(zrl->zr_refcount >= 0);
zrl->zr_refcount++;
#ifdef ZFS_DEBUG
zrl->zr_owner = curthread;
zrl->zr_caller = zc;
#endif
mutex_exit(&zrl->zr_mtx);
}
inline bool compare_and_swap(PtrT *ptr, const ValueT& comp_val, const ValueT& exchange_val)
{
#if ELEVELDB_IS_SOLARIS
return (1==atomic_cas_32(ptr, comp_val, exchange_val));
#else
return __sync_bool_compare_and_swap(ptr, comp_val, exchange_val);
#endif
}
inline bool compare_and_swap(volatile uint32_t *ptr, const int& comp_val, const int& exchange_val)
{
#if ELEVELDB_IS_SOLARIS
return ((uint32_t) comp_val==atomic_cas_32(ptr, comp_val, exchange_val));
#else
return __sync_bool_compare_and_swap(ptr, comp_val, exchange_val);
#endif
}
void
db_resume_others(void)
{
int cpu_me = cpu_number();
if (atomic_cas_32(&ddb_cpu, cpu_me, NOCPU) == cpu_me)
mp_resume_cpus();
}
bool add_ref_lock() // true on success
{
for( ;; )
{
uint32_t tmp = static_cast< uint32_t const volatile& >( use_count_ );
if( tmp == 0 ) return false;
if( atomic_cas_32( &use_count_, tmp, tmp + 1 ) == tmp ) return true;
}
}
bool mcs_rwlock::attempt_read()
{
unsigned int old_value = *&_holders;
if(old_value & WRITER ||
old_value != atomic_cas_32(&_holders, old_value, old_value+READER))
return false;
membar_enter();
return true;
}
bool
__atomic_compare_exchange_4(volatile uint32_t *mem,
uint32_t *expected, uint32_t desired,
bool weak, int success, int failure)
{
uint32_t old = *expected;
/*
* Ignore the details (weak, memory model on success and failure)
* and just do the cas. If we get here the compiler couldn't
* do better and it mostly will not matter at all.
*/
return atomic_cas_32(mem, old, desired) == old;
}
static int
db_suspend_others(void)
{
int cpu_me = cpu_number();
bool win;
if (cpus == NULL)
return 1;
win = atomic_cas_32(&ddb_cpu, NOCPU, cpu_me) == (uint32_t)NOCPU;
if (win)
mp_pause_cpus();
return win;
}
/*
* This locking needs work and will misbehave severely if:
* 1) the backing memory has to be paged in
* 2) some lockholder exits while holding the lock
*/
static void
shmif_lockbus(struct shmif_mem *busmem)
{
int i = 0;
while (__predict_false(atomic_cas_32(&busmem->shm_lock,
LOCK_UNLOCKED, LOCK_LOCKED) == LOCK_LOCKED)) {
if (__predict_false(++i > LOCK_COOLDOWN)) {
/* wait 1ms */
rumpuser_clock_sleep(RUMPUSER_CLOCK_RELWALL,
0, 1000*1000);
i = 0;
}
continue;
}
membar_enter();
}
/*
* Attempt to release a writer lock. Return true on success.
*/
static int
write_unlock_try(rwlock_t *rwlp)
{
volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
uint32_t readers;
ulwp_t *self = curthread;
no_preempt(self);
while (((readers = *rwstate) & URW_HAS_WAITERS) == 0) {
if (atomic_cas_32(rwstate, readers, 0) == readers) {
preempt(self);
return (1);
}
}
preempt(self);
return (0);
}
int lk_lock(struct lock* l)
{
#ifdef LOCK_FREE
for (;;)
{
int r = atomic_cas_32(&l->l, 0, 1);
if (r == 0)
{
break;
}
}
return 0;
#else
return pthread_mutex_lock(&l->l);
#endif
}
/*
* Lock-less utility that scans through logical slot array looking for a
* free slot to reuse.
*/
static struct slot *
get_free_slot(pthread_var_np_t vp)
{
struct slots *slots;
struct slot *slot;
size_t i;
for (slots = atomic_read_ptr((volatile void **)&vp->slots);
slots != NULL;
slots = atomic_read_ptr((volatile void **)&slots->next)) {
for (i = 0; i < slots->slot_count; i++) {
slot = &slots->slot_array[i];
if (atomic_cas_32(&slot->in_use, 0, 1) == 0)
return slot;
}
}
return NULL;
}
int
oce_atomic_reserve(uint32_t *count_p, uint32_t n)
{
uint32_t oldval;
uint32_t newval;
/*
* ATOMICALLY
*/
do {
oldval = *count_p;
if (oldval < n)
return (-1);
newval = oldval - n;
} while (atomic_cas_32(count_p, oldval, newval) != oldval);
return (newval);
}
/*
* Attempt to acquire a readers lock. Return true on success.
*/
static int
read_lock_try(rwlock_t *rwlp, int ignore_waiters_flag)
{
volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
uint32_t mask = ignore_waiters_flag?
URW_WRITE_LOCKED : (URW_HAS_WAITERS | URW_WRITE_LOCKED);
uint32_t readers;
ulwp_t *self = curthread;
no_preempt(self);
while (((readers = *rwstate) & mask) == 0) {
if (atomic_cas_32(rwstate, readers, readers + 1) == readers) {
preempt(self);
return (1);
}
}
preempt(self);
return (0);
}
/*
* Adjust the memnode config after a DR operation.
*
* It is rather tricky to do these updates since we can't
* protect the memnode structures with locks, so we must
* be mindful of the order in which updates and reads to
* these values can occur.
*/
void
mem_node_add_slice(pfn_t start, pfn_t end)
{
int mnode;
mnodeset_t newmask, oldmask;
/*
* DR will pass us the first pfn that is allocatable.
* We need to round down to get the real start of
* the slice.
*/
if (mem_node_physalign) {
start &= ~(btop(mem_node_physalign) - 1);
end = roundup(end, btop(mem_node_physalign)) - 1;
}
mnode = PFN_2_MEM_NODE(start);
ASSERT(mnode < max_mem_nodes);
if (atomic_cas_32((uint32_t *)&mem_node_config[mnode].exists, 0, 1)) {
/*
* Add slice to existing node.
*/
if (start < mem_node_config[mnode].physbase)
mem_node_config[mnode].physbase = start;
if (end > mem_node_config[mnode].physmax)
mem_node_config[mnode].physmax = end;
} else {
mem_node_config[mnode].physbase = start;
mem_node_config[mnode].physmax = end;
atomic_inc_16(&num_memnodes);
do {
oldmask = memnodes_mask;
newmask = memnodes_mask | (1ull << mnode);
} while (atomic_cas_64(&memnodes_mask, oldmask, newmask) !=
oldmask);
}
/*
* Let the common lgrp framework know about the new memory
*/
lgrp_config(LGRP_CONFIG_MEM_ADD, mnode, MEM_NODE_2_LGRPHAND(mnode));
}
int
zrl_tryenter(zrlock_t *zrl)
{
uint32_t n = (uint32_t)zrl->zr_refcount;
if (n == 0) {
uint32_t cas = atomic_cas_32(
(uint32_t *)&zrl->zr_refcount, 0, ZRL_LOCKED);
if (cas == 0) {
#ifdef ZFS_DEBUG
ASSERT(zrl->zr_owner == NULL);
zrl->zr_owner = curthread;
#endif
return (1);
}
}
ASSERT((int32_t)n > ZRL_DESTROYED);
return (0);
}
/*ARGSUSED*/
static void
fipe_idle_exit(void* arg, cpu_idle_callback_context_t ctx, int flags)
{
uint32_t cnt;
hrtime_t ts;
struct fipe_cpu_state *sp;
sp = &fipe_cpu_states[CPU->cpu_id];
if (sp->cond_ready) {
do {
cnt = fipe_gbl_ctrl.cpu_count;
ASSERT(cnt > 0);
} while (atomic_cas_32(&fipe_gbl_ctrl.cpu_count,
cnt, cnt - 1) != cnt);
/*
* Try to disable power saving state.
* Only the first CPU waking from idle state will try to
* disable power saving state, all other CPUs will just go
* on and not try to wait for memory to recover from power
* saving state.
* So there are possible periods during which some CPUs are in
* active state but memory is in power saving state.
* This is OK, since it is an uncommon case, and it is
* better for performance to let them continue as their
* blocking latency is smaller than a mutex, and is only
* hit in the uncommon condition.
*/
if (cnt == ncpus) {
fipe_disable();
ts = cpu_idle_prop_get_hrtime(fipe_idle_ctrl.prop_exit,
ctx);
fipe_gbl_ctrl.time_in_pm += ts - fipe_gbl_ctrl.enter_ts;
}
}
}
bool mcs_rwlock::_attempt_write(unsigned int expected)
{
/* succeeds iff we are the only reader (if expected==READER)
* or if there are no readers or writers (if expected==0)
*
* How do we know there's the only reader is us?
* A: we rely on these facts: this is called with expected==READER only
* from attempt_upgrade(), which is called from latch only in the case
* in which we hold the latch in LATCH_SH mode and are requesting it in LATCH_EX mode.
If there is a writer waiting we have to get in line like everyone else.
No need for a membar because we already hold the latch
*/
ext_qnode me = QUEUE_EXT_QNODE_INITIALIZER;
if(*&_holders != expected || !attempt(&me))
return false;
// at this point, we've called mcs_lock::attempt(&me), and
// have acquired the parent/mcs lock
// The following line replaces our reader bit with a writer bit.
bool result = (expected == atomic_cas_32(&_holders, expected, WRITER));
release(me); // parent/mcs lock
membar_enter();
return result;
}
static int
hxge_start(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp)
{
int dma_status, status = 0;
p_tx_desc_t tx_desc_ring_vp;
hpi_handle_t hpi_desc_handle;
hxge_os_dma_handle_t tx_desc_dma_handle;
p_tx_desc_t tx_desc_p;
p_tx_msg_t tx_msg_ring;
p_tx_msg_t tx_msg_p;
tx_desc_t tx_desc, *tmp_desc_p;
tx_desc_t sop_tx_desc, *sop_tx_desc_p;
p_tx_pkt_header_t hdrp;
p_tx_pkt_hdr_all_t pkthdrp;
uint8_t npads = 0;
uint64_t dma_ioaddr;
uint32_t dma_flags;
int last_bidx;
uint8_t *b_rptr;
caddr_t kaddr;
uint32_t nmblks;
uint32_t ngathers;
uint32_t clen;
int len;
uint32_t pkt_len, pack_len, min_len;
uint32_t bcopy_thresh;
int i, cur_index, sop_index;
uint16_t tail_index;
boolean_t tail_wrap = B_FALSE;
hxge_dma_common_t desc_area;
hxge_os_dma_handle_t dma_handle;
ddi_dma_cookie_t dma_cookie;
hpi_handle_t hpi_handle;
p_mblk_t nmp;
p_mblk_t t_mp;
uint32_t ncookies;
boolean_t good_packet;
boolean_t mark_mode = B_FALSE;
p_hxge_stats_t statsp;
p_hxge_tx_ring_stats_t tdc_stats;
t_uscalar_t start_offset = 0;
t_uscalar_t stuff_offset = 0;
t_uscalar_t end_offset = 0;
t_uscalar_t value = 0;
t_uscalar_t cksum_flags = 0;
boolean_t cksum_on = B_FALSE;
uint32_t boff = 0;
uint64_t tot_xfer_len = 0, tmp_len = 0;
boolean_t header_set = B_FALSE;
tdc_tdr_kick_t kick;
uint32_t offset;
#ifdef HXGE_DEBUG
p_tx_desc_t tx_desc_ring_pp;
p_tx_desc_t tx_desc_pp;
tx_desc_t *save_desc_p;
int dump_len;
int sad_len;
uint64_t sad;
int xfer_len;
uint32_t msgsize;
#endif
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start: tx dma channel %d", tx_ring_p->tdc));
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start: Starting tdc %d desc pending %d",
tx_ring_p->tdc, tx_ring_p->descs_pending));
statsp = hxgep->statsp;
if (hxgep->statsp->port_stats.lb_mode == hxge_lb_normal) {
if (!statsp->mac_stats.link_up) {
freemsg(mp);
HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: "
"link not up or LB mode"));
goto hxge_start_fail1;
}
}
mac_hcksum_get(mp, &start_offset, &stuff_offset, &end_offset, &value,
&cksum_flags);
if (!HXGE_IS_VLAN_PACKET(mp->b_rptr)) {
start_offset += sizeof (ether_header_t);
stuff_offset += sizeof (ether_header_t);
} else {
start_offset += sizeof (struct ether_vlan_header);
stuff_offset += sizeof (struct ether_vlan_header);
}
if (cksum_flags & HCK_PARTIALCKSUM) {
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start: mp $%p len %d "
"cksum_flags 0x%x (partial checksum) ",
mp, MBLKL(mp), cksum_flags));
cksum_on = B_TRUE;
}
MUTEX_ENTER(&tx_ring_p->lock);
start_again:
ngathers = 0;
sop_index = tx_ring_p->wr_index;
#ifdef HXGE_DEBUG
if (tx_ring_p->descs_pending) {
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start: desc pending %d ",
tx_ring_p->descs_pending));
}
dump_len = (int)(MBLKL(mp));
dump_len = (dump_len > 128) ? 128: dump_len;
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start: tdc %d: dumping ...: b_rptr $%p "
"(Before header reserve: ORIGINAL LEN %d)",
tx_ring_p->tdc, mp->b_rptr, dump_len));
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start: dump packets (IP ORIGINAL b_rptr $%p): %s",
mp->b_rptr, hxge_dump_packet((char *)mp->b_rptr, dump_len)));
#endif
tdc_stats = tx_ring_p->tdc_stats;
mark_mode = (tx_ring_p->descs_pending &&
((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) <
hxge_tx_minfree));
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"TX Descriptor ring is channel %d mark mode %d",
tx_ring_p->tdc, mark_mode));
if (!hxge_txdma_reclaim(hxgep, tx_ring_p, hxge_tx_minfree)) {
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"TX Descriptor ring is full: channel %d", tx_ring_p->tdc));
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"TX Descriptor ring is full: channel %d", tx_ring_p->tdc));
(void) atomic_cas_32((uint32_t *)&tx_ring_p->queueing, 0, 1);
tdc_stats->tx_no_desc++;
MUTEX_EXIT(&tx_ring_p->lock);
status = 1;
goto hxge_start_fail1;
}
nmp = mp;
i = sop_index = tx_ring_p->wr_index;
nmblks = 0;
ngathers = 0;
pkt_len = 0;
pack_len = 0;
clen = 0;
last_bidx = -1;
good_packet = B_TRUE;
desc_area = tx_ring_p->tdc_desc;
hpi_handle = desc_area.hpi_handle;
hpi_desc_handle.regh = (hxge_os_acc_handle_t)
DMA_COMMON_ACC_HANDLE(desc_area);
hpi_desc_handle.hxgep = hxgep;
tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
#ifdef HXGE_DEBUG
#if defined(__i386)
tx_desc_ring_pp = (p_tx_desc_t)(uint32_t)DMA_COMMON_IOADDR(desc_area);
#else
tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area);
#endif
#endif
tx_desc_dma_handle = (hxge_os_dma_handle_t)DMA_COMMON_HANDLE(desc_area);
tx_msg_ring = tx_ring_p->tx_msg_ring;
HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: wr_index %d i %d",
sop_index, i));
#ifdef HXGE_DEBUG
msgsize = msgdsize(nmp);
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start(1): wr_index %d i %d msgdsize %d",
sop_index, i, msgsize));
#endif
/*
* The first 16 bytes of the premapped buffer are reserved
* for header. No padding will be used.
*/
pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE;
if (hxge_tx_use_bcopy) {
bcopy_thresh = (hxge_bcopy_thresh - TX_PKT_HEADER_SIZE);
} else {
bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE);
}
while (nmp) {
good_packet = B_TRUE;
b_rptr = nmp->b_rptr;
len = MBLKL(nmp);
if (len <= 0) {
nmp = nmp->b_cont;
continue;
}
nmblks++;
HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(1): nmblks %d "
"len %d pkt_len %d pack_len %d",
nmblks, len, pkt_len, pack_len));
/*
* Hardware limits the transfer length to 4K.
* If len is more than 4K, we need to break
* nmp into two chunks: Make first chunk smaller
* than 4K. The second chunk will be broken into
* less than 4K (if needed) during the next pass.
*/
if (len > (TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE)) {
if ((t_mp = dupb(nmp)) != NULL) {
nmp->b_wptr = nmp->b_rptr +
(TX_MAX_TRANSFER_LENGTH -
TX_PKT_HEADER_SIZE);
t_mp->b_rptr = nmp->b_wptr;
t_mp->b_cont = nmp->b_cont;
nmp->b_cont = t_mp;
len = MBLKL(nmp);
} else {
good_packet = B_FALSE;
goto hxge_start_fail2;
}
}
tx_desc.value = 0;
tx_desc_p = &tx_desc_ring_vp[i];
#ifdef HXGE_DEBUG
tx_desc_pp = &tx_desc_ring_pp[i];
#endif
tx_msg_p = &tx_msg_ring[i];
#if defined(__i386)
hpi_desc_handle.regp = (uint32_t)tx_desc_p;
#else
hpi_desc_handle.regp = (uint64_t)tx_desc_p;
#endif
if (!header_set &&
((!hxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) ||
(len >= bcopy_thresh))) {
header_set = B_TRUE;
bcopy_thresh += TX_PKT_HEADER_SIZE;
boff = 0;
pack_len = 0;
kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
hdrp = (p_tx_pkt_header_t)kaddr;
clen = pkt_len;
dma_handle = tx_msg_p->buf_dma_handle;
dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
offset = tx_msg_p->offset_index * hxge_bcopy_thresh;
(void) ddi_dma_sync(dma_handle,
offset, hxge_bcopy_thresh, DDI_DMA_SYNC_FORDEV);
tx_msg_p->flags.dma_type = USE_BCOPY;
goto hxge_start_control_header_only;
}
pkt_len += len;
pack_len += len;
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start(3): desc entry %d DESC IOADDR $%p "
"desc_vp $%p tx_desc_p $%p desc_pp $%p tx_desc_pp $%p "
"len %d pkt_len %d pack_len %d",
i,
DMA_COMMON_IOADDR(desc_area),
tx_desc_ring_vp, tx_desc_p,
tx_desc_ring_pp, tx_desc_pp,
len, pkt_len, pack_len));
if (len < bcopy_thresh) {
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start(4): USE BCOPY: "));
if (hxge_tx_tiny_pack) {
uint32_t blst = TXDMA_DESC_NEXT_INDEX(i, -1,
tx_ring_p->tx_wrap_mask);
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start(5): pack"));
if ((pack_len <= bcopy_thresh) &&
(last_bidx == blst)) {
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start: pack(6) "
"(pkt_len %d pack_len %d)",
pkt_len, pack_len));
i = blst;
tx_desc_p = &tx_desc_ring_vp[i];
#ifdef HXGE_DEBUG
tx_desc_pp = &tx_desc_ring_pp[i];
#endif
tx_msg_p = &tx_msg_ring[i];
boff = pack_len - len;
ngathers--;
} else if (pack_len > bcopy_thresh &&
header_set) {
pack_len = len;
boff = 0;
bcopy_thresh = hxge_bcopy_thresh;
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start(7): > max NEW "
"bcopy thresh %d "
"pkt_len %d pack_len %d(next)",
bcopy_thresh, pkt_len, pack_len));
}
last_bidx = i;
}
kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) {
hdrp = (p_tx_pkt_header_t)kaddr;
header_set = B_TRUE;
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start(7_x2): "
"pkt_len %d pack_len %d (new hdrp $%p)",
pkt_len, pack_len, hdrp));
}
tx_msg_p->flags.dma_type = USE_BCOPY;
kaddr += boff;
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start(8): USE BCOPY: before bcopy "
"DESC IOADDR $%p entry %d bcopy packets %d "
"bcopy kaddr $%p bcopy ioaddr (SAD) $%p "
"bcopy clen %d bcopy boff %d",
DMA_COMMON_IOADDR(desc_area), i,
tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr,
clen, boff));
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start: 1USE BCOPY: "));
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start: 2USE BCOPY: "));
HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: "
"last USE BCOPY: copy from b_rptr $%p "
"to KADDR $%p (len %d offset %d",
b_rptr, kaddr, len, boff));
bcopy(b_rptr, kaddr, len);
#ifdef HXGE_DEBUG
dump_len = (len > 128) ? 128: len;
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start: dump packets "
"(After BCOPY len %d)"
"(b_rptr $%p): %s", len, nmp->b_rptr,
hxge_dump_packet((char *)nmp->b_rptr,
dump_len)));
#endif
dma_handle = tx_msg_p->buf_dma_handle;
dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
offset = tx_msg_p->offset_index * hxge_bcopy_thresh;
(void) ddi_dma_sync(dma_handle,
offset, hxge_bcopy_thresh, DDI_DMA_SYNC_FORDEV);
clen = len + boff;
tdc_stats->tx_hdr_pkts++;
HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(9): "
"USE BCOPY: DESC IOADDR $%p entry %d "
"bcopy packets %d bcopy kaddr $%p "
"bcopy ioaddr (SAD) $%p bcopy clen %d "
"bcopy boff %d",
DMA_COMMON_IOADDR(desc_area), i,
tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr,
clen, boff));
} else {
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start(12): USE DVMA: len %d", len));
tx_msg_p->flags.dma_type = USE_DMA;
dma_flags = DDI_DMA_WRITE;
if (len < hxge_dma_stream_thresh) {
dma_flags |= DDI_DMA_CONSISTENT;
} else {
dma_flags |= DDI_DMA_STREAMING;
}
dma_handle = tx_msg_p->dma_handle;
dma_status = ddi_dma_addr_bind_handle(dma_handle, NULL,
(caddr_t)b_rptr, len, dma_flags,
DDI_DMA_DONTWAIT, NULL,
&dma_cookie, &ncookies);
if (dma_status == DDI_DMA_MAPPED) {
dma_ioaddr = dma_cookie.dmac_laddress;
len = (int)dma_cookie.dmac_size;
clen = (uint32_t)dma_cookie.dmac_size;
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start(12_1): "
"USE DVMA: len %d clen %d ngathers %d",
len, clen, ngathers));
#if defined(__i386)
hpi_desc_handle.regp = (uint32_t)tx_desc_p;
#else
hpi_desc_handle.regp = (uint64_t)tx_desc_p;
#endif
while (ncookies > 1) {
ngathers++;
/*
* this is the fix for multiple
* cookies, which are basically
* a descriptor entry, we don't set
* SOP bit as well as related fields
*/
(void) hpi_txdma_desc_gather_set(
hpi_desc_handle, &tx_desc,
(ngathers -1), mark_mode,
ngathers, dma_ioaddr, clen);
tx_msg_p->tx_msg_size = clen;
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start: DMA "
"ncookie %d ngathers %d "
"dma_ioaddr $%p len %d"
"desc $%p descp $%p (%d)",
ncookies, ngathers,
dma_ioaddr, clen,
*tx_desc_p, tx_desc_p, i));
ddi_dma_nextcookie(dma_handle,
&dma_cookie);
dma_ioaddr = dma_cookie.dmac_laddress;
len = (int)dma_cookie.dmac_size;
clen = (uint32_t)dma_cookie.dmac_size;
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start(12_2): "
"USE DVMA: len %d clen %d ",
len, clen));
i = TXDMA_DESC_NEXT_INDEX(i, 1,
tx_ring_p->tx_wrap_mask);
tx_desc_p = &tx_desc_ring_vp[i];
hpi_desc_handle.regp =
#if defined(__i386)
(uint32_t)tx_desc_p;
#else
(uint64_t)tx_desc_p;
#endif
tx_msg_p = &tx_msg_ring[i];
tx_msg_p->flags.dma_type = USE_NONE;
tx_desc.value = 0;
ncookies--;
}
tdc_stats->tx_ddi_pkts++;
HXGE_DEBUG_MSG((hxgep, TX_CTL,
"==> hxge_start: DMA: ddi packets %d",
tdc_stats->tx_ddi_pkts));
} else {
HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
"dma mapping failed for %d "
"bytes addr $%p flags %x (%d)",
len, b_rptr, status, status));
good_packet = B_FALSE;
tdc_stats->tx_dma_bind_fail++;
tx_msg_p->flags.dma_type = USE_NONE;
status = 1;
goto hxge_start_fail2;
}
} /* ddi dvma */
nmp = nmp->b_cont;
hxge_start_control_header_only:
#if defined(__i386)
hpi_desc_handle.regp = (uint32_t)tx_desc_p;
#else
hpi_desc_handle.regp = (uint64_t)tx_desc_p;
#endif
ngathers++;
if (ngathers == 1) {
#ifdef HXGE_DEBUG
save_desc_p = &sop_tx_desc;
#endif
sop_tx_desc_p = &sop_tx_desc;
sop_tx_desc_p->value = 0;
sop_tx_desc_p->bits.tr_len = clen;
sop_tx_desc_p->bits.sad = dma_ioaddr >> 32;
sop_tx_desc_p->bits.sad_l = dma_ioaddr & 0xffffffff;
} else {
static inline int32_t
fenced_compare_exchange_strong_32(int32_t *ptr, int32_t expected, int32_t desired)
{
return atomic_cas_32((volatile unsigned int*)ptr, expected, desired);
}
template<typename T> static T cas(volatile T *ptr, T oldval, T newval) { return atomic_cas_32(ptr, oldval, newval); }
/*ARGSUSED*/
static void
fipe_idle_enter(void *arg, cpu_idle_callback_context_t ctx,
cpu_idle_check_wakeup_t check_func, void* check_arg)
{
hrtime_t ts;
uint32_t cnt;
uint64_t iowait;
cpu_t *cp = CPU;
struct fipe_cpu_state *sp;
sp = &fipe_cpu_states[cp->cpu_id];
ts = cpu_idle_prop_get_hrtime(fipe_idle_ctrl.prop_enter, ctx);
if (fipe_pm_policy != FIPE_PM_POLICY_DISABLE &&
fipe_ioat_ctrl.ioat_ready &&
sp->state_ready && sp->throttle_ts <= ts) {
/* Adjust iowait count for local CPU. */
iowait = CPU_STATS(cp, sys.iowait);
if (iowait != sp->last_iowait) {
atomic_add_64(&fipe_gbl_ctrl.io_waiters,
iowait - sp->last_iowait);
sp->last_iowait = iowait;
}
/* Check current CPU status. */
if (fipe_check_cpu(sp, ctx, ts)) {
/* Increase count of CPU ready for power saving. */
do {
cnt = fipe_gbl_ctrl.cpu_count;
ASSERT(cnt < ncpus);
} while (atomic_cas_32(&fipe_gbl_ctrl.cpu_count,
cnt, cnt + 1) != cnt);
/*
* Enable power saving if all CPUs are idle.
*/
if (cnt + 1 == ncpus) {
if (fipe_gbl_ctrl.io_waiters == 0) {
fipe_gbl_ctrl.enter_ts = ts;
fipe_enable(fipe_pm_throttle_level,
check_func, check_arg);
/* There are ongoing block io operations. */
} else {
FIPE_KSTAT_DETAIL_INC(bio_busy_cnt);
}
}
}
} else if (fipe_pm_policy == FIPE_PM_POLICY_DISABLE ||
fipe_ioat_ctrl.ioat_ready == B_FALSE) {
if (sp->cond_ready == B_TRUE) {
sp->cond_ready = B_FALSE;
}
} else if (sp->state_ready == B_FALSE) {
sp->cond_ready = B_FALSE;
sp->state_ready = B_TRUE;
sp->throttle_ts = 0;
sp->next_ts = ts + fipe_idle_ctrl.tick_interval;
sp->last_busy = cpu_idle_prop_get_hrtime(
fipe_idle_ctrl.prop_busy, ctx);
sp->last_idle = cpu_idle_prop_get_hrtime(
fipe_idle_ctrl.prop_idle, ctx);
sp->last_intr = cpu_idle_prop_get_hrtime(
fipe_idle_ctrl.prop_intr, ctx);
sp->idle_count = 0;
}
}
static int
futex_wake_op_execute(int32_t *addr, int32_t val3)
{
int32_t op = FUTEX_OP_OP(val3);
int32_t cmp = FUTEX_OP_CMP(val3);
int32_t cmparg = FUTEX_OP_CMPARG(val3);
int32_t oparg, oldval, newval;
label_t ljb;
int rval;
if ((uintptr_t)addr >= KERNELBASE)
return (set_errno(EFAULT));
if (on_fault(&ljb))
return (set_errno(EFAULT));
oparg = FUTEX_OP_OPARG(val3);
do {
oldval = *addr;
newval = oparg;
switch (op) {
case FUTEX_OP_SET:
break;
case FUTEX_OP_ADD:
newval += oparg;
break;
case FUTEX_OP_OR:
newval |= oparg;
break;
case FUTEX_OP_ANDN:
newval &= ~oparg;
break;
case FUTEX_OP_XOR:
newval ^= oparg;
break;
default:
no_fault();
return (set_errno(EINVAL));
}
} while (atomic_cas_32((uint32_t *)addr, oldval, newval) != oldval);
no_fault();
switch (cmp) {
case FUTEX_OP_CMP_EQ:
rval = (oldval == cmparg);
break;
case FUTEX_OP_CMP_NE:
rval = (oldval != cmparg);
break;
case FUTEX_OP_CMP_LT:
rval = (oldval < cmparg);
break;
case FUTEX_OP_CMP_LE:
rval = (oldval <= cmparg);
break;
case FUTEX_OP_CMP_GT:
rval = (oldval > cmparg);
break;
case FUTEX_OP_CMP_GE:
rval = (oldval >= cmparg);
break;
default:
return (set_errno(EINVAL));
}
return (rval);
}
