void zrl_add_impl(zrlock_t *zrl, const char *zc) { for (;;) { uint32_t n = (uint32_t)zrl->zr_refcount; while (n != ZRL_LOCKED) { uint32_t cas = atomic_cas_32( (uint32_t *)&zrl->zr_refcount, n, n + 1); if (cas == n) { ASSERT3S((int32_t)n, >=, 0); #ifdef ZFS_DEBUG if (zrl->zr_owner == curthread) { DTRACE_PROBE2(zrlock__reentry, zrlock_t *, zrl, uint32_t, n); } zrl->zr_owner = curthread; zrl->zr_caller = zc; #endif return; } n = cas; } mutex_enter(&zrl->zr_mtx); while (zrl->zr_refcount == ZRL_LOCKED) { cv_wait(&zrl->zr_cv, &zrl->zr_mtx); } mutex_exit(&zrl->zr_mtx); }
/* * Allocate an unassigned memnode. */ int mem_node_alloc() { int mnode; mnodeset_t newmask, oldmask; /* * Find an unused memnode. Update it atomically to prevent * a first time memnode creation race. */ for (mnode = 0; mnode < max_mem_nodes; mnode++) if (atomic_cas_32((uint32_t *)&mem_node_config[mnode].exists, 0, 1) == 0) break; if (mnode >= max_mem_nodes) panic("Out of free memnodes\n"); mem_node_config[mnode].physbase = (uint64_t)-1; mem_node_config[mnode].physmax = 0; atomic_inc_16(&num_memnodes); do { oldmask = memnodes_mask; newmask = memnodes_mask | (1ull << mnode); } while (atomic_cas_64(&memnodes_mask, oldmask, newmask) != oldmask); return (mnode); }
zrl_add(zrlock_t *zrl) #endif { uint32_t n = (uint32_t)zrl->zr_refcount; while (n != ZRL_LOCKED) { uint32_t cas = atomic_cas_32( (uint32_t *)&zrl->zr_refcount, n, n + 1); if (cas == n) { ASSERT((int32_t)n >= 0); #ifdef ZFS_DEBUG if (zrl->zr_owner == curthread) { DTRACE_PROBE2(zrlock__reentry, zrlock_t *, zrl, uint32_t, n); } zrl->zr_owner = curthread; zrl->zr_caller = zc; #endif return; } n = cas; } mutex_enter(&zrl->zr_mtx); while (zrl->zr_refcount == ZRL_LOCKED) { cv_wait(&zrl->zr_cv, &zrl->zr_mtx); } ASSERT(zrl->zr_refcount >= 0); zrl->zr_refcount++; #ifdef ZFS_DEBUG zrl->zr_owner = curthread; zrl->zr_caller = zc; #endif mutex_exit(&zrl->zr_mtx); }
inline bool compare_and_swap(PtrT *ptr, const ValueT& comp_val, const ValueT& exchange_val) { #if ELEVELDB_IS_SOLARIS return (1==atomic_cas_32(ptr, comp_val, exchange_val)); #else return __sync_bool_compare_and_swap(ptr, comp_val, exchange_val); #endif }
inline bool compare_and_swap(volatile uint32_t *ptr, const int& comp_val, const int& exchange_val) { #if ELEVELDB_IS_SOLARIS return ((uint32_t) comp_val==atomic_cas_32(ptr, comp_val, exchange_val)); #else return __sync_bool_compare_and_swap(ptr, comp_val, exchange_val); #endif }
void db_resume_others(void) { int cpu_me = cpu_number(); if (atomic_cas_32(&ddb_cpu, cpu_me, NOCPU) == cpu_me) mp_resume_cpus(); }
bool add_ref_lock() // true on success { for( ;; ) { uint32_t tmp = static_cast< uint32_t const volatile& >( use_count_ ); if( tmp == 0 ) return false; if( atomic_cas_32( &use_count_, tmp, tmp + 1 ) == tmp ) return true; } }
bool mcs_rwlock::attempt_read() { unsigned int old_value = *&_holders; if(old_value & WRITER || old_value != atomic_cas_32(&_holders, old_value, old_value+READER)) return false; membar_enter(); return true; }
bool __atomic_compare_exchange_4(volatile uint32_t *mem, uint32_t *expected, uint32_t desired, bool weak, int success, int failure) { uint32_t old = *expected; /* * Ignore the details (weak, memory model on success and failure) * and just do the cas. If we get here the compiler couldn't * do better and it mostly will not matter at all. */ return atomic_cas_32(mem, old, desired) == old; }
static int db_suspend_others(void) { int cpu_me = cpu_number(); bool win; if (cpus == NULL) return 1; win = atomic_cas_32(&ddb_cpu, NOCPU, cpu_me) == (uint32_t)NOCPU; if (win) mp_pause_cpus(); return win; }
/* * This locking needs work and will misbehave severely if: * 1) the backing memory has to be paged in * 2) some lockholder exits while holding the lock */ static void shmif_lockbus(struct shmif_mem *busmem) { int i = 0; while (__predict_false(atomic_cas_32(&busmem->shm_lock, LOCK_UNLOCKED, LOCK_LOCKED) == LOCK_LOCKED)) { if (__predict_false(++i > LOCK_COOLDOWN)) { /* wait 1ms */ rumpuser_clock_sleep(RUMPUSER_CLOCK_RELWALL, 0, 1000*1000); i = 0; } continue; } membar_enter(); }
/* * Attempt to release a writer lock. Return true on success. */ static int write_unlock_try(rwlock_t *rwlp) { volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers; uint32_t readers; ulwp_t *self = curthread; no_preempt(self); while (((readers = *rwstate) & URW_HAS_WAITERS) == 0) { if (atomic_cas_32(rwstate, readers, 0) == readers) { preempt(self); return (1); } } preempt(self); return (0); }
int lk_lock(struct lock* l) { #ifdef LOCK_FREE for (;;) { int r = atomic_cas_32(&l->l, 0, 1); if (r == 0) { break; } } return 0; #else return pthread_mutex_lock(&l->l); #endif }
/* * Lock-less utility that scans through logical slot array looking for a * free slot to reuse. */ static struct slot * get_free_slot(pthread_var_np_t vp) { struct slots *slots; struct slot *slot; size_t i; for (slots = atomic_read_ptr((volatile void **)&vp->slots); slots != NULL; slots = atomic_read_ptr((volatile void **)&slots->next)) { for (i = 0; i < slots->slot_count; i++) { slot = &slots->slot_array[i]; if (atomic_cas_32(&slot->in_use, 0, 1) == 0) return slot; } } return NULL; }
int oce_atomic_reserve(uint32_t *count_p, uint32_t n) { uint32_t oldval; uint32_t newval; /* * ATOMICALLY */ do { oldval = *count_p; if (oldval < n) return (-1); newval = oldval - n; } while (atomic_cas_32(count_p, oldval, newval) != oldval); return (newval); }
/* * Attempt to acquire a readers lock. Return true on success. */ static int read_lock_try(rwlock_t *rwlp, int ignore_waiters_flag) { volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers; uint32_t mask = ignore_waiters_flag? URW_WRITE_LOCKED : (URW_HAS_WAITERS | URW_WRITE_LOCKED); uint32_t readers; ulwp_t *self = curthread; no_preempt(self); while (((readers = *rwstate) & mask) == 0) { if (atomic_cas_32(rwstate, readers, readers + 1) == readers) { preempt(self); return (1); } } preempt(self); return (0); }
/* * Adjust the memnode config after a DR operation. * * It is rather tricky to do these updates since we can't * protect the memnode structures with locks, so we must * be mindful of the order in which updates and reads to * these values can occur. */ void mem_node_add_slice(pfn_t start, pfn_t end) { int mnode; mnodeset_t newmask, oldmask; /* * DR will pass us the first pfn that is allocatable. * We need to round down to get the real start of * the slice. */ if (mem_node_physalign) { start &= ~(btop(mem_node_physalign) - 1); end = roundup(end, btop(mem_node_physalign)) - 1; } mnode = PFN_2_MEM_NODE(start); ASSERT(mnode < max_mem_nodes); if (atomic_cas_32((uint32_t *)&mem_node_config[mnode].exists, 0, 1)) { /* * Add slice to existing node. */ if (start < mem_node_config[mnode].physbase) mem_node_config[mnode].physbase = start; if (end > mem_node_config[mnode].physmax) mem_node_config[mnode].physmax = end; } else { mem_node_config[mnode].physbase = start; mem_node_config[mnode].physmax = end; atomic_inc_16(&num_memnodes); do { oldmask = memnodes_mask; newmask = memnodes_mask | (1ull << mnode); } while (atomic_cas_64(&memnodes_mask, oldmask, newmask) != oldmask); } /* * Let the common lgrp framework know about the new memory */ lgrp_config(LGRP_CONFIG_MEM_ADD, mnode, MEM_NODE_2_LGRPHAND(mnode)); }
int zrl_tryenter(zrlock_t *zrl) { uint32_t n = (uint32_t)zrl->zr_refcount; if (n == 0) { uint32_t cas = atomic_cas_32( (uint32_t *)&zrl->zr_refcount, 0, ZRL_LOCKED); if (cas == 0) { #ifdef ZFS_DEBUG ASSERT(zrl->zr_owner == NULL); zrl->zr_owner = curthread; #endif return (1); } } ASSERT((int32_t)n > ZRL_DESTROYED); return (0); }
/*ARGSUSED*/ static void fipe_idle_exit(void* arg, cpu_idle_callback_context_t ctx, int flags) { uint32_t cnt; hrtime_t ts; struct fipe_cpu_state *sp; sp = &fipe_cpu_states[CPU->cpu_id]; if (sp->cond_ready) { do { cnt = fipe_gbl_ctrl.cpu_count; ASSERT(cnt > 0); } while (atomic_cas_32(&fipe_gbl_ctrl.cpu_count, cnt, cnt - 1) != cnt); /* * Try to disable power saving state. * Only the first CPU waking from idle state will try to * disable power saving state, all other CPUs will just go * on and not try to wait for memory to recover from power * saving state. * So there are possible periods during which some CPUs are in * active state but memory is in power saving state. * This is OK, since it is an uncommon case, and it is * better for performance to let them continue as their * blocking latency is smaller than a mutex, and is only * hit in the uncommon condition. */ if (cnt == ncpus) { fipe_disable(); ts = cpu_idle_prop_get_hrtime(fipe_idle_ctrl.prop_exit, ctx); fipe_gbl_ctrl.time_in_pm += ts - fipe_gbl_ctrl.enter_ts; } } }
bool mcs_rwlock::_attempt_write(unsigned int expected) { /* succeeds iff we are the only reader (if expected==READER) * or if there are no readers or writers (if expected==0) * * How do we know there's the only reader is us? * A: we rely on these facts: this is called with expected==READER only * from attempt_upgrade(), which is called from latch only in the case * in which we hold the latch in LATCH_SH mode and are requesting it in LATCH_EX mode. If there is a writer waiting we have to get in line like everyone else. No need for a membar because we already hold the latch */ ext_qnode me = QUEUE_EXT_QNODE_INITIALIZER; if(*&_holders != expected || !attempt(&me)) return false; // at this point, we've called mcs_lock::attempt(&me), and // have acquired the parent/mcs lock // The following line replaces our reader bit with a writer bit. bool result = (expected == atomic_cas_32(&_holders, expected, WRITER)); release(me); // parent/mcs lock membar_enter(); return result; }
static int hxge_start(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) { int dma_status, status = 0; p_tx_desc_t tx_desc_ring_vp; hpi_handle_t hpi_desc_handle; hxge_os_dma_handle_t tx_desc_dma_handle; p_tx_desc_t tx_desc_p; p_tx_msg_t tx_msg_ring; p_tx_msg_t tx_msg_p; tx_desc_t tx_desc, *tmp_desc_p; tx_desc_t sop_tx_desc, *sop_tx_desc_p; p_tx_pkt_header_t hdrp; p_tx_pkt_hdr_all_t pkthdrp; uint8_t npads = 0; uint64_t dma_ioaddr; uint32_t dma_flags; int last_bidx; uint8_t *b_rptr; caddr_t kaddr; uint32_t nmblks; uint32_t ngathers; uint32_t clen; int len; uint32_t pkt_len, pack_len, min_len; uint32_t bcopy_thresh; int i, cur_index, sop_index; uint16_t tail_index; boolean_t tail_wrap = B_FALSE; hxge_dma_common_t desc_area; hxge_os_dma_handle_t dma_handle; ddi_dma_cookie_t dma_cookie; hpi_handle_t hpi_handle; p_mblk_t nmp; p_mblk_t t_mp; uint32_t ncookies; boolean_t good_packet; boolean_t mark_mode = B_FALSE; p_hxge_stats_t statsp; p_hxge_tx_ring_stats_t tdc_stats; t_uscalar_t start_offset = 0; t_uscalar_t stuff_offset = 0; t_uscalar_t end_offset = 0; t_uscalar_t value = 0; t_uscalar_t cksum_flags = 0; boolean_t cksum_on = B_FALSE; uint32_t boff = 0; uint64_t tot_xfer_len = 0, tmp_len = 0; boolean_t header_set = B_FALSE; tdc_tdr_kick_t kick; uint32_t offset; #ifdef HXGE_DEBUG p_tx_desc_t tx_desc_ring_pp; p_tx_desc_t tx_desc_pp; tx_desc_t *save_desc_p; int dump_len; int sad_len; uint64_t sad; int xfer_len; uint32_t msgsize; #endif HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: tx dma channel %d", tx_ring_p->tdc)); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: Starting tdc %d desc pending %d", tx_ring_p->tdc, tx_ring_p->descs_pending)); statsp = hxgep->statsp; if (hxgep->statsp->port_stats.lb_mode == hxge_lb_normal) { if (!statsp->mac_stats.link_up) { freemsg(mp); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: " "link not up or LB mode")); goto hxge_start_fail1; } } mac_hcksum_get(mp, &start_offset, &stuff_offset, &end_offset, &value, &cksum_flags); if (!HXGE_IS_VLAN_PACKET(mp->b_rptr)) { start_offset += sizeof (ether_header_t); stuff_offset += sizeof (ether_header_t); } else { start_offset += sizeof (struct ether_vlan_header); stuff_offset += sizeof (struct ether_vlan_header); } if (cksum_flags & HCK_PARTIALCKSUM) { HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: mp $%p len %d " "cksum_flags 0x%x (partial checksum) ", mp, MBLKL(mp), cksum_flags)); cksum_on = B_TRUE; } MUTEX_ENTER(&tx_ring_p->lock); start_again: ngathers = 0; sop_index = tx_ring_p->wr_index; #ifdef HXGE_DEBUG if (tx_ring_p->descs_pending) { HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: desc pending %d ", tx_ring_p->descs_pending)); } dump_len = (int)(MBLKL(mp)); dump_len = (dump_len > 128) ? 128: dump_len; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: tdc %d: dumping ...: b_rptr $%p " "(Before header reserve: ORIGINAL LEN %d)", tx_ring_p->tdc, mp->b_rptr, dump_len)); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: dump packets (IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, hxge_dump_packet((char *)mp->b_rptr, dump_len))); #endif tdc_stats = tx_ring_p->tdc_stats; mark_mode = (tx_ring_p->descs_pending && ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) < hxge_tx_minfree)); HXGE_DEBUG_MSG((hxgep, TX_CTL, "TX Descriptor ring is channel %d mark mode %d", tx_ring_p->tdc, mark_mode)); if (!hxge_txdma_reclaim(hxgep, tx_ring_p, hxge_tx_minfree)) { HXGE_DEBUG_MSG((hxgep, TX_CTL, "TX Descriptor ring is full: channel %d", tx_ring_p->tdc)); HXGE_DEBUG_MSG((hxgep, TX_CTL, "TX Descriptor ring is full: channel %d", tx_ring_p->tdc)); (void) atomic_cas_32((uint32_t *)&tx_ring_p->queueing, 0, 1); tdc_stats->tx_no_desc++; MUTEX_EXIT(&tx_ring_p->lock); status = 1; goto hxge_start_fail1; } nmp = mp; i = sop_index = tx_ring_p->wr_index; nmblks = 0; ngathers = 0; pkt_len = 0; pack_len = 0; clen = 0; last_bidx = -1; good_packet = B_TRUE; desc_area = tx_ring_p->tdc_desc; hpi_handle = desc_area.hpi_handle; hpi_desc_handle.regh = (hxge_os_acc_handle_t) DMA_COMMON_ACC_HANDLE(desc_area); hpi_desc_handle.hxgep = hxgep; tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); #ifdef HXGE_DEBUG #if defined(__i386) tx_desc_ring_pp = (p_tx_desc_t)(uint32_t)DMA_COMMON_IOADDR(desc_area); #else tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area); #endif #endif tx_desc_dma_handle = (hxge_os_dma_handle_t)DMA_COMMON_HANDLE(desc_area); tx_msg_ring = tx_ring_p->tx_msg_ring; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: wr_index %d i %d", sop_index, i)); #ifdef HXGE_DEBUG msgsize = msgdsize(nmp); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(1): wr_index %d i %d msgdsize %d", sop_index, i, msgsize)); #endif /* * The first 16 bytes of the premapped buffer are reserved * for header. No padding will be used. */ pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; if (hxge_tx_use_bcopy) { bcopy_thresh = (hxge_bcopy_thresh - TX_PKT_HEADER_SIZE); } else { bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); } while (nmp) { good_packet = B_TRUE; b_rptr = nmp->b_rptr; len = MBLKL(nmp); if (len <= 0) { nmp = nmp->b_cont; continue; } nmblks++; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(1): nmblks %d " "len %d pkt_len %d pack_len %d", nmblks, len, pkt_len, pack_len)); /* * Hardware limits the transfer length to 4K. * If len is more than 4K, we need to break * nmp into two chunks: Make first chunk smaller * than 4K. The second chunk will be broken into * less than 4K (if needed) during the next pass. */ if (len > (TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE)) { if ((t_mp = dupb(nmp)) != NULL) { nmp->b_wptr = nmp->b_rptr + (TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE); t_mp->b_rptr = nmp->b_wptr; t_mp->b_cont = nmp->b_cont; nmp->b_cont = t_mp; len = MBLKL(nmp); } else { good_packet = B_FALSE; goto hxge_start_fail2; } } tx_desc.value = 0; tx_desc_p = &tx_desc_ring_vp[i]; #ifdef HXGE_DEBUG tx_desc_pp = &tx_desc_ring_pp[i]; #endif tx_msg_p = &tx_msg_ring[i]; #if defined(__i386) hpi_desc_handle.regp = (uint32_t)tx_desc_p; #else hpi_desc_handle.regp = (uint64_t)tx_desc_p; #endif if (!header_set && ((!hxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || (len >= bcopy_thresh))) { header_set = B_TRUE; bcopy_thresh += TX_PKT_HEADER_SIZE; boff = 0; pack_len = 0; kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); hdrp = (p_tx_pkt_header_t)kaddr; clen = pkt_len; dma_handle = tx_msg_p->buf_dma_handle; dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); offset = tx_msg_p->offset_index * hxge_bcopy_thresh; (void) ddi_dma_sync(dma_handle, offset, hxge_bcopy_thresh, DDI_DMA_SYNC_FORDEV); tx_msg_p->flags.dma_type = USE_BCOPY; goto hxge_start_control_header_only; } pkt_len += len; pack_len += len; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(3): desc entry %d DESC IOADDR $%p " "desc_vp $%p tx_desc_p $%p desc_pp $%p tx_desc_pp $%p " "len %d pkt_len %d pack_len %d", i, DMA_COMMON_IOADDR(desc_area), tx_desc_ring_vp, tx_desc_p, tx_desc_ring_pp, tx_desc_pp, len, pkt_len, pack_len)); if (len < bcopy_thresh) { HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(4): USE BCOPY: ")); if (hxge_tx_tiny_pack) { uint32_t blst = TXDMA_DESC_NEXT_INDEX(i, -1, tx_ring_p->tx_wrap_mask); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(5): pack")); if ((pack_len <= bcopy_thresh) && (last_bidx == blst)) { HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: pack(6) " "(pkt_len %d pack_len %d)", pkt_len, pack_len)); i = blst; tx_desc_p = &tx_desc_ring_vp[i]; #ifdef HXGE_DEBUG tx_desc_pp = &tx_desc_ring_pp[i]; #endif tx_msg_p = &tx_msg_ring[i]; boff = pack_len - len; ngathers--; } else if (pack_len > bcopy_thresh && header_set) { pack_len = len; boff = 0; bcopy_thresh = hxge_bcopy_thresh; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(7): > max NEW " "bcopy thresh %d " "pkt_len %d pack_len %d(next)", bcopy_thresh, pkt_len, pack_len)); } last_bidx = i; } kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { hdrp = (p_tx_pkt_header_t)kaddr; header_set = B_TRUE; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(7_x2): " "pkt_len %d pack_len %d (new hdrp $%p)", pkt_len, pack_len, hdrp)); } tx_msg_p->flags.dma_type = USE_BCOPY; kaddr += boff; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(8): USE BCOPY: before bcopy " "DESC IOADDR $%p entry %d bcopy packets %d " "bcopy kaddr $%p bcopy ioaddr (SAD) $%p " "bcopy clen %d bcopy boff %d", DMA_COMMON_IOADDR(desc_area), i, tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr, clen, boff)); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: 1USE BCOPY: ")); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: 2USE BCOPY: ")); HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: " "last USE BCOPY: copy from b_rptr $%p " "to KADDR $%p (len %d offset %d", b_rptr, kaddr, len, boff)); bcopy(b_rptr, kaddr, len); #ifdef HXGE_DEBUG dump_len = (len > 128) ? 128: len; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: dump packets " "(After BCOPY len %d)" "(b_rptr $%p): %s", len, nmp->b_rptr, hxge_dump_packet((char *)nmp->b_rptr, dump_len))); #endif dma_handle = tx_msg_p->buf_dma_handle; dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); offset = tx_msg_p->offset_index * hxge_bcopy_thresh; (void) ddi_dma_sync(dma_handle, offset, hxge_bcopy_thresh, DDI_DMA_SYNC_FORDEV); clen = len + boff; tdc_stats->tx_hdr_pkts++; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(9): " "USE BCOPY: DESC IOADDR $%p entry %d " "bcopy packets %d bcopy kaddr $%p " "bcopy ioaddr (SAD) $%p bcopy clen %d " "bcopy boff %d", DMA_COMMON_IOADDR(desc_area), i, tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr, clen, boff)); } else { HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(12): USE DVMA: len %d", len)); tx_msg_p->flags.dma_type = USE_DMA; dma_flags = DDI_DMA_WRITE; if (len < hxge_dma_stream_thresh) { dma_flags |= DDI_DMA_CONSISTENT; } else { dma_flags |= DDI_DMA_STREAMING; } dma_handle = tx_msg_p->dma_handle; dma_status = ddi_dma_addr_bind_handle(dma_handle, NULL, (caddr_t)b_rptr, len, dma_flags, DDI_DMA_DONTWAIT, NULL, &dma_cookie, &ncookies); if (dma_status == DDI_DMA_MAPPED) { dma_ioaddr = dma_cookie.dmac_laddress; len = (int)dma_cookie.dmac_size; clen = (uint32_t)dma_cookie.dmac_size; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(12_1): " "USE DVMA: len %d clen %d ngathers %d", len, clen, ngathers)); #if defined(__i386) hpi_desc_handle.regp = (uint32_t)tx_desc_p; #else hpi_desc_handle.regp = (uint64_t)tx_desc_p; #endif while (ncookies > 1) { ngathers++; /* * this is the fix for multiple * cookies, which are basically * a descriptor entry, we don't set * SOP bit as well as related fields */ (void) hpi_txdma_desc_gather_set( hpi_desc_handle, &tx_desc, (ngathers -1), mark_mode, ngathers, dma_ioaddr, clen); tx_msg_p->tx_msg_size = clen; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: DMA " "ncookie %d ngathers %d " "dma_ioaddr $%p len %d" "desc $%p descp $%p (%d)", ncookies, ngathers, dma_ioaddr, clen, *tx_desc_p, tx_desc_p, i)); ddi_dma_nextcookie(dma_handle, &dma_cookie); dma_ioaddr = dma_cookie.dmac_laddress; len = (int)dma_cookie.dmac_size; clen = (uint32_t)dma_cookie.dmac_size; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(12_2): " "USE DVMA: len %d clen %d ", len, clen)); i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); tx_desc_p = &tx_desc_ring_vp[i]; hpi_desc_handle.regp = #if defined(__i386) (uint32_t)tx_desc_p; #else (uint64_t)tx_desc_p; #endif tx_msg_p = &tx_msg_ring[i]; tx_msg_p->flags.dma_type = USE_NONE; tx_desc.value = 0; ncookies--; } tdc_stats->tx_ddi_pkts++; HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: DMA: ddi packets %d", tdc_stats->tx_ddi_pkts)); } else { HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "dma mapping failed for %d " "bytes addr $%p flags %x (%d)", len, b_rptr, status, status)); good_packet = B_FALSE; tdc_stats->tx_dma_bind_fail++; tx_msg_p->flags.dma_type = USE_NONE; status = 1; goto hxge_start_fail2; } } /* ddi dvma */ nmp = nmp->b_cont; hxge_start_control_header_only: #if defined(__i386) hpi_desc_handle.regp = (uint32_t)tx_desc_p; #else hpi_desc_handle.regp = (uint64_t)tx_desc_p; #endif ngathers++; if (ngathers == 1) { #ifdef HXGE_DEBUG save_desc_p = &sop_tx_desc; #endif sop_tx_desc_p = &sop_tx_desc; sop_tx_desc_p->value = 0; sop_tx_desc_p->bits.tr_len = clen; sop_tx_desc_p->bits.sad = dma_ioaddr >> 32; sop_tx_desc_p->bits.sad_l = dma_ioaddr & 0xffffffff; } else {
static inline int32_t fenced_compare_exchange_strong_32(int32_t *ptr, int32_t expected, int32_t desired) { return atomic_cas_32((volatile unsigned int*)ptr, expected, desired); }
template<typename T> static T cas(volatile T *ptr, T oldval, T newval) { return atomic_cas_32(ptr, oldval, newval); }
/*ARGSUSED*/ static void fipe_idle_enter(void *arg, cpu_idle_callback_context_t ctx, cpu_idle_check_wakeup_t check_func, void* check_arg) { hrtime_t ts; uint32_t cnt; uint64_t iowait; cpu_t *cp = CPU; struct fipe_cpu_state *sp; sp = &fipe_cpu_states[cp->cpu_id]; ts = cpu_idle_prop_get_hrtime(fipe_idle_ctrl.prop_enter, ctx); if (fipe_pm_policy != FIPE_PM_POLICY_DISABLE && fipe_ioat_ctrl.ioat_ready && sp->state_ready && sp->throttle_ts <= ts) { /* Adjust iowait count for local CPU. */ iowait = CPU_STATS(cp, sys.iowait); if (iowait != sp->last_iowait) { atomic_add_64(&fipe_gbl_ctrl.io_waiters, iowait - sp->last_iowait); sp->last_iowait = iowait; } /* Check current CPU status. */ if (fipe_check_cpu(sp, ctx, ts)) { /* Increase count of CPU ready for power saving. */ do { cnt = fipe_gbl_ctrl.cpu_count; ASSERT(cnt < ncpus); } while (atomic_cas_32(&fipe_gbl_ctrl.cpu_count, cnt, cnt + 1) != cnt); /* * Enable power saving if all CPUs are idle. */ if (cnt + 1 == ncpus) { if (fipe_gbl_ctrl.io_waiters == 0) { fipe_gbl_ctrl.enter_ts = ts; fipe_enable(fipe_pm_throttle_level, check_func, check_arg); /* There are ongoing block io operations. */ } else { FIPE_KSTAT_DETAIL_INC(bio_busy_cnt); } } } } else if (fipe_pm_policy == FIPE_PM_POLICY_DISABLE || fipe_ioat_ctrl.ioat_ready == B_FALSE) { if (sp->cond_ready == B_TRUE) { sp->cond_ready = B_FALSE; } } else if (sp->state_ready == B_FALSE) { sp->cond_ready = B_FALSE; sp->state_ready = B_TRUE; sp->throttle_ts = 0; sp->next_ts = ts + fipe_idle_ctrl.tick_interval; sp->last_busy = cpu_idle_prop_get_hrtime( fipe_idle_ctrl.prop_busy, ctx); sp->last_idle = cpu_idle_prop_get_hrtime( fipe_idle_ctrl.prop_idle, ctx); sp->last_intr = cpu_idle_prop_get_hrtime( fipe_idle_ctrl.prop_intr, ctx); sp->idle_count = 0; } }
static int futex_wake_op_execute(int32_t *addr, int32_t val3) { int32_t op = FUTEX_OP_OP(val3); int32_t cmp = FUTEX_OP_CMP(val3); int32_t cmparg = FUTEX_OP_CMPARG(val3); int32_t oparg, oldval, newval; label_t ljb; int rval; if ((uintptr_t)addr >= KERNELBASE) return (set_errno(EFAULT)); if (on_fault(&ljb)) return (set_errno(EFAULT)); oparg = FUTEX_OP_OPARG(val3); do { oldval = *addr; newval = oparg; switch (op) { case FUTEX_OP_SET: break; case FUTEX_OP_ADD: newval += oparg; break; case FUTEX_OP_OR: newval |= oparg; break; case FUTEX_OP_ANDN: newval &= ~oparg; break; case FUTEX_OP_XOR: newval ^= oparg; break; default: no_fault(); return (set_errno(EINVAL)); } } while (atomic_cas_32((uint32_t *)addr, oldval, newval) != oldval); no_fault(); switch (cmp) { case FUTEX_OP_CMP_EQ: rval = (oldval == cmparg); break; case FUTEX_OP_CMP_NE: rval = (oldval != cmparg); break; case FUTEX_OP_CMP_LT: rval = (oldval < cmparg); break; case FUTEX_OP_CMP_LE: rval = (oldval <= cmparg); break; case FUTEX_OP_CMP_GT: rval = (oldval > cmparg); break; case FUTEX_OP_CMP_GE: rval = (oldval >= cmparg); break; default: return (set_errno(EINVAL)); } return (rval); }