static inline SeFastpathRet octeon_se_fastpath_transform_check_pmtu(SeFastpathPacketContext pc, size_t packet_out_len) { if (cvmx_unlikely(packet_out_len > 65535)) { OCTEON_SE_DEBUG(3, "Dropping packet because of overflow\n"); return OCTEON_SE_FASTPATH_RET_DROP; } if (cvmx_unlikely((packet_out_len > pc->mtu) #ifdef OCTEON_SE_FASTPATH_FRAGMENTATION && (pc->s->ipv4_df || pc->s->ip_version_6) #endif /* OCTEON_SE_FASTPATH_FRAGMENTATION */ )) { OCTEON_SE_DEBUG(5, "Need to send ICMP error message, " "passing to slowpath\n"); return OCTEON_SE_FASTPATH_RET_SLOWPATH; } return OCTEON_SE_FASTPATH_RET_OK; }
/** * Get clock rate based on the clock type. * * @param node - CPU node number * @param clock - Enumeration of the clock type. * @return - return the clock rate. */ uint64_t cvmx_clock_get_rate_node(int node, cvmx_clock_t clock) { const uint64_t REF_CLOCK = 50000000; #ifdef CVMX_BUILD_FOR_UBOOT uint64_t rate_eclk = 0; uint64_t rate_sclk = 0; uint64_t rate_dclk = 0; #endif if (cvmx_unlikely(!rate_eclk)) { /* Note: The order of these checks is important. ** octeon_has_feature(OCTEON_FEATURE_PCIE) is true for both 6XXX ** and 52XX/56XX, so OCTEON_FEATURE_NPEI _must_ be checked first */ if (octeon_has_feature(OCTEON_FEATURE_NPEI)) { cvmx_npei_dbg_data_t npei_dbg_data; npei_dbg_data.u64 = cvmx_read_csr(CVMX_PEXP_NPEI_DBG_DATA); rate_eclk = REF_CLOCK * npei_dbg_data.s.c_mul; rate_sclk = rate_eclk; } else if (OCTEON_IS_OCTEON3()) { cvmx_rst_boot_t rst_boot; rst_boot.u64 = cvmx_read_csr_node(node, CVMX_RST_BOOT); rate_eclk = REF_CLOCK * rst_boot.s.c_mul; rate_sclk = REF_CLOCK * rst_boot.s.pnr_mul; } else if (octeon_has_feature(OCTEON_FEATURE_PCIE)) { cvmx_mio_rst_boot_t mio_rst_boot; mio_rst_boot.u64 = cvmx_read_csr(CVMX_MIO_RST_BOOT); rate_eclk = REF_CLOCK * mio_rst_boot.s.c_mul; rate_sclk = REF_CLOCK * mio_rst_boot.s.pnr_mul; } else { cvmx_dbg_data_t dbg_data; dbg_data.u64 = cvmx_read_csr(CVMX_DBG_DATA); rate_eclk = REF_CLOCK * dbg_data.s.c_mul; rate_sclk = rate_eclk; } } switch (clock) { case CVMX_CLOCK_SCLK: case CVMX_CLOCK_TIM: case CVMX_CLOCK_IPD: return rate_sclk; case CVMX_CLOCK_RCLK: case CVMX_CLOCK_CORE: return rate_eclk; case CVMX_CLOCK_DDR: #if !defined(CVMX_BUILD_FOR_LINUX_HOST) && !defined(CVMX_BUILD_FOR_TOOLCHAIN) if (cvmx_unlikely(!rate_dclk)) rate_dclk = cvmx_sysinfo_get()->dram_data_rate_hz; #endif return rate_dclk; } cvmx_dprintf("cvmx_clock_get_rate: Unknown clock type\n"); return 0; }
/** * @INTERNAL * Calls the user supplied callback when an event happens. * * @param usb USB device state populated by * cvmx_usbd_initialize(). * @param reason Reason for the callback * @param endpoint_num * Endpoint number * @param bytes_transferred * Bytes transferred */ static void __cvmx_usbd_callback(cvmx_usbd_state_t *usb, cvmx_usbd_callback_t reason, int endpoint_num, int bytes_transferred) { if (usb->callback[reason]) { if (cvmx_unlikely(usb->init_flags & CVMX_USBD_INITIALIZE_FLAGS_DEBUG)) cvmx_dprintf("%s: Calling callback reason=%d endpoint=%d bytes=%d func=%p data=%p\n", __FUNCTION__, reason, endpoint_num, bytes_transferred, usb->callback[reason], usb->callback_data[reason]); usb->callback[reason](reason, endpoint_num, bytes_transferred, usb->callback_data[reason]); } else { if (cvmx_unlikely(usb->init_flags & CVMX_USBD_INITIALIZE_FLAGS_DEBUG)) cvmx_dprintf("%s: No callback for reason=%d endpoint=%d bytes=%d\n", __FUNCTION__, reason, endpoint_num, bytes_transferred); } }
/** * @INTERNAL * Poll a device mode endpoint for status * * @param usb USB device state populated by * cvmx_usbd_initialize(). * @param endpoint_num * Endpoint to poll * * @return Zero on success */ static int __cvmx_usbd_poll_in_endpoint(cvmx_usbd_state_t *usb, int endpoint_num) { cvmx_usbcx_diepintx_t usbc_diepint; if (cvmx_unlikely(usb->init_flags & CVMX_USBD_INITIALIZE_FLAGS_DEBUG)) cvmx_dprintf("%s: endpoint=%d\n", __FUNCTION__, endpoint_num); usbc_diepint.u32 = __cvmx_usbd_read_csr32(usb, CVMX_USBCX_DIEPINTX(endpoint_num, usb->index)); __cvmx_usbd_write_csr32(usb, CVMX_USBCX_DIEPINTX(endpoint_num, usb->index), usbc_diepint.u32); if (usbc_diepint.s.epdisbld) { /* Endpoint Disabled Interrupt (EPDisbld) This bit indicates that the endpoint is disabled per the application's request. */ /* Nothing to do */ } if (usbc_diepint.s.xfercompl) { cvmx_usbcx_dieptsizx_t usbc_dieptsiz; int bytes_transferred; /* Transfer Completed Interrupt (XferCompl) Indicates that the programmed transfer is complete on the AHB as well as on the USB, for this endpoint. */ usbc_dieptsiz.u32 = __cvmx_usbd_read_csr32(usb, CVMX_USBCX_DIEPTSIZX(endpoint_num, usb->index)); bytes_transferred = usb->endpoint[endpoint_num].buffer_length - usbc_dieptsiz.s.xfersize; __cvmx_usbd_callback(usb, CVMX_USBD_CALLBACK_IN_COMPLETE, endpoint_num, bytes_transferred); } return 0; }
/** * Register a callback function to process USB events * * @param usb USB device state populated by * cvmx_usbd_initialize(). * @param reason The reason this callback should be called * @param func Function to call * @param user_data User supplied data for the callback * * @return Zero on succes, negative on failure */ int cvmx_usbd_register(cvmx_usbd_state_t *usb, cvmx_usbd_callback_t reason, cvmx_usbd_callback_func_t func, void *user_data) { if (cvmx_unlikely(usb->init_flags & CVMX_USBD_INITIALIZE_FLAGS_DEBUG)) cvmx_dprintf("%s: Register reason=%d func=%p data=%p\n", __FUNCTION__, reason, func, user_data); usb->callback[reason] = func; usb->callback_data[reason] = user_data; return 0; }
/** * Disable an OUT endpoint * * @param usb USB device state populated by * cvmx_usbd_initialize(). * @param endpoint_num * Endpoint number to disable * * @return Zero on success, negative on failure */ int cvmx_usbd_out_endpoint_disable(cvmx_usbd_state_t *usb, int endpoint_num) { cvmx_usbcx_doepctlx_t usbc_doepctl; if (cvmx_unlikely(usb->init_flags & CVMX_USBD_INITIALIZE_FLAGS_DEBUG)) cvmx_dprintf("%s: endpoint=%d\n", __FUNCTION__, endpoint_num); usbc_doepctl.u32 = __cvmx_usbd_read_csr32(usb, CVMX_USBCX_DOEPCTLX(endpoint_num, usb->index)); if (usbc_doepctl.s.epena && !usbc_doepctl.s.epdis) { usbc_doepctl.s.epdis = 1; __cvmx_usbd_write_csr32(usb, CVMX_USBCX_DOEPCTLX(endpoint_num, usb->index), usbc_doepctl.u32); } return 0; }
/** * Shutdown a USB port after a call to cvmx_usbd_initialize(). * * @param usb USB device state populated by * cvmx_usbd_initialize(). * * @return Zero or a negative on error. */ int cvmx_usbd_shutdown(cvmx_usbd_state_t *usb) { cvmx_usbnx_clk_ctl_t usbn_clk_ctl; if (cvmx_unlikely(usb->init_flags & CVMX_USBD_INITIALIZE_FLAGS_DEBUG)) cvmx_dprintf("%s: Called\n", __FUNCTION__); /* Disable the clocks and put them in power on reset */ usbn_clk_ctl.u64 = cvmx_read_csr(CVMX_USBNX_CLK_CTL(usb->index)); usbn_clk_ctl.s.enable = 1; usbn_clk_ctl.s.por = 1; usbn_clk_ctl.s.hclk_rst = 1; usbn_clk_ctl.s.prst = 0; usbn_clk_ctl.s.hrst = 0; cvmx_write_csr(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64); return 0; }
void S3_send_packet(cvmx_wqe_t * work) { uint64_t port; cvmx_buf_ptr_t packet_ptr; cvmx_pko_command_word0_t pko_command; /* Build a PKO pointer to this packet */ pko_command.u64 = 0; /* Errata PKI-100 fix. We need to fix chain pointers on segmneted packets. Although the size is also wrong on a single buffer packet, PKO doesn't care so we ignore it */ if (cvmx_unlikely(work->word2.s.bufs > 1)) cvmx_helper_fix_ipd_packet_chain(work); port = work->ipprt; if( port >= portbase + portnum) port = work->ipprt - portnum; else port = work->ipprt + portnum; int queue = cvmx_pko_get_base_queue(port); cvmx_pko_send_packet_prepare(port, queue, CVMX_PKO_LOCK_ATOMIC_TAG); pko_command.s.total_bytes = work->len; pko_command.s.segs = work->word2.s.bufs; pko_command.s.ipoffp1 = 14 + 1; packet_ptr = work->packet_ptr; //cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, 0); cvm_common_free_fpa_buffer(work, CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE / CVMX_CACHE_LINE_SIZE); work = NULL; /* * Send the packet and wait for the tag switch to complete before * accessing the output queue. This ensures the locking required * for the queue. * */ if (cvmx_pko_send_packet_finish(port, queue, pko_command, packet_ptr, CVMX_PKO_LOCK_ATOMIC_TAG)) { printf("Failed to send packet using cvmx_pko_send_packet_finish\ n"); } }
/** * @INTERNAL * Poll a device mode endpoint for status * * @param usb USB device state populated by * cvmx_usbd_initialize(). * @param endpoint_num * Endpoint to poll * * @return Zero on success */ static int __cvmx_usbd_poll_out_endpoint(cvmx_usbd_state_t *usb, int endpoint_num) { cvmx_usbcx_doepintx_t usbc_doepint; if (cvmx_unlikely(usb->init_flags & CVMX_USBD_INITIALIZE_FLAGS_DEBUG)) cvmx_dprintf("%s: endpoint=%d\n", __FUNCTION__, endpoint_num); usbc_doepint.u32 = __cvmx_usbd_read_csr32(usb, CVMX_USBCX_DOEPINTX(endpoint_num, usb->index)); __cvmx_usbd_write_csr32(usb, CVMX_USBCX_DOEPINTX(endpoint_num, usb->index), usbc_doepint.u32); if (usbc_doepint.s.setup) { /* SETUP Phase Done (SetUp) Applies to control OUT endpoints only. Indicates that the SETUP phase for the control endpoint is complete and no more back-to-back SETUP packets were received for the current control transfer. On this interrupt, the application can decode the received SETUP data packet. */ __cvmx_usbd_callback(usb, CVMX_USBD_CALLBACK_DEVICE_SETUP, endpoint_num, 0); } if (usbc_doepint.s.epdisbld) { /* Endpoint Disabled Interrupt (EPDisbld) This bit indicates that the endpoint is disabled per the application's request. */ /* Nothing to do */ } if (usbc_doepint.s.xfercompl) { cvmx_usbcx_doeptsizx_t usbc_doeptsiz; int bytes_transferred; /* Transfer Completed Interrupt (XferCompl) Indicates that the programmed transfer is complete on the AHB as well as on the USB, for this endpoint. */ usbc_doeptsiz.u32 = __cvmx_usbd_read_csr32(usb, CVMX_USBCX_DOEPTSIZX(endpoint_num, usb->index)); bytes_transferred = usb->endpoint[endpoint_num].buffer_length - usbc_doeptsiz.s.xfersize; __cvmx_usbd_callback(usb, CVMX_USBD_CALLBACK_OUT_COMPLETE, endpoint_num, bytes_transferred); } return 0; }
uint32_t octeon_se_fastpath_fragc_init(SeFastpathCoreContext core, SeFastpath fastpath, SeFastpathFragmentContext fragc, SeFastpathPacketContext pc, size_t mtu, uint8_t df_on_first_fragment) { uint8_t * header; size_t packet_len = pc->s->ip_len; /* Initialize common fields in the fragment context. */ fragc->pc = pc; fragc->mtu = mtu; fragc->offset = 0; /* Get a pointer to the packet to be fragmented. */ header = (uint8_t *)cvmx_phys_to_ptr(pc->wqe->packet_ptr.s.addr) + pc->s->ip_offset; if (pc->s->ip_version_6) { uint16_t frag_hlen; uint16_t frag_data_len; fragc->total_len = packet_len - OCTEON_SE_FASTPATH_IP6_HDRLEN; /* Compute fragments' header and data lengths. */ frag_hlen = OCTEON_SE_FASTPATH_IP6_HDRLEN + OCTEON_SE_FASTPATH_IP6_EXT_FRAG_HDRLEN; frag_data_len = ((size_t) (mtu - frag_hlen)) & (size_t) ~7; OCTEON_SE_ASSERT((frag_data_len > 0) && (frag_data_len <= (65535 - frag_hlen))); /* Store that information into the fragmentation context. */ fragc->frag_hlen = frag_hlen; fragc->frag_data_len = frag_data_len; OCTEON_SE_FASTPATH_GET_NEXT_IPV6_FRAG_ID(core, fragc->u.ipv6.id); memcpy(fragc->u.ipv6.frag_hdr, header, OCTEON_SE_FASTPATH_IP6_HDRLEN); octeon_se_fastpath_packet_buffer_create(fragc->original_pkt, pc->wqe->packet_ptr, pc->s->ip_offset + OCTEON_SE_FASTPATH_IP6_HDRLEN, fragc->total_len, pc->wqe->word2.s.bufs); } else { /* Check if the packet has DF bit set. */ if (cvmx_unlikely(pc->s->ipv4_df)) { OCTEON_SE_DEBUG(7, "Cannot fragment packet. DF bit is set\n"); return 1; } fragc->total_len = packet_len - OCTEON_SE_FASTPATH_IP4_HDRLEN; fragc->frag_hlen = OCTEON_SE_FASTPATH_IP4_HDRLEN; /* Compute amount of data to go in fragments. */ fragc->frag_data_len = ((size_t)(mtu - OCTEON_SE_FASTPATH_IP4_HDRLEN)) & (size_t) ~7; OCTEON_SE_ASSERT(fragc->frag_data_len > 0 && fragc->frag_data_len < 65535); fragc->u.ipv4.df_on_first_fragment = df_on_first_fragment; /* Store computed values into the fragmentation context. */ memcpy(fragc->u.ipv4.frag_hdr, header, OCTEON_SE_FASTPATH_IP4_HDRLEN); octeon_se_fastpath_packet_buffer_create(fragc->original_pkt, pc->wqe->packet_ptr, pc->s->ip_offset + OCTEON_SE_FASTPATH_IP4_HDRLEN, fragc->total_len, pc->wqe->word2.s.bufs); } return 0; }
static uint32_t octeon_se_fastpath_fragc_helper_alloc(SeFastpathFragmentContext fragc, SeFastpathPacketContext orig_pc, SeFastpathPacketContext frag_pc, uint16_t data_len) { cvmx_wqe_t *wqe; cvmx_buf_ptr_t fragment; uint64_t num_segments = 0; uint32_t len; size_t alignment; wqe = cvmx_fpa_alloc(CVMX_FPA_WQE_POOL); if (cvmx_unlikely(wqe == NULL)) { OCTEON_SE_DEBUG(3, "Out of memory while allocating wqe for fragment.\n"); return 1; } len = data_len + fragc->frag_hlen; if (cvmx_unlikely(orig_pc->s->ip_version_6)) alignment = (OCTEON_SE_ALIGN_64(orig_pc->s->ip_offset + OCTEON_SE_FASTPATH_IP6_HDRLEN + OCTEON_SE_FASTPATH_IP6_EXT_FRAG_HDRLEN) - (orig_pc->s->ip_offset + OCTEON_SE_FASTPATH_IP6_HDRLEN + OCTEON_SE_FASTPATH_IP6_EXT_FRAG_HDRLEN)); else alignment = (OCTEON_SE_ALIGN_64(orig_pc->s->ip_offset + OCTEON_SE_FASTPATH_IP4_HDRLEN) - (orig_pc->s->ip_offset + OCTEON_SE_FASTPATH_IP4_HDRLEN)); fragment.u64 = octeon_se_fastpath_alloc_packet_chain(len + orig_pc->s->ip_offset, alignment, &num_segments); if (cvmx_unlikely(fragment.u64 == 0)) { OCTEON_SE_DEBUG(3, "Out of memory while allocating fragments.\n"); cvmx_fpa_free(wqe, CVMX_FPA_WQE_POOL, 0); return 1; } wqe->packet_ptr.u64 = fragment.u64; wqe->len = len + orig_pc->s->ip_offset; wqe->word2.s.bufs = num_segments; frag_pc->wqe = wqe; frag_pc->s->ip_offset = orig_pc->s->ip_offset; frag_pc->s->ip_len = len; frag_pc->s->ip_version_6 = orig_pc->s->ip_version_6; frag_pc->mtu = orig_pc->mtu; frag_pc->oport = orig_pc->oport; frag_pc->nh_index = orig_pc->nh_index; frag_pc->media_hdrlen = orig_pc->media_hdrlen; memcpy(frag_pc->media_hdr.data, orig_pc->media_hdr.data, frag_pc->media_hdrlen); return 0; }
SeFastpathPacketContext octeon_se_fastpath_fragc_next(SeFastpathCoreContext core, SeFastpath fastpath, SeFastpathFragmentContext fragc) { SeFastpathPacketContext frag; SeFastpathPacketBufferStruct fragment_buffer[1]; uint8_t * header; cvmx_buf_ptr_t packet_out; uint16_t hlen, data_len, len, offset_orig; uint16_t fragoff_orig, fragoff, checksum; uint8_t is_last_frag; /* If an error caused pc to be freed, return NULL to indicate we are done. */ if (fragc->pc == NULL || fragc->offset >= fragc->total_len) return NULL; hlen = fragc->frag_hlen; data_len = fragc->frag_data_len; /* Determine the length of the data section of the fragment. */ if (fragc->offset + data_len < fragc->total_len) len = data_len; else len = fragc->total_len - fragc->offset; if (fragc->offset + len == fragc->total_len) is_last_frag = TRUE; else is_last_frag = FALSE; OCTEON_SE_DEBUG(7, "Sending fragment offset=%d, len=%d\n", fragc->offset, len); /* Allocate packet context and state for the fragment. */ frag = &core->fragment.s; memset(frag, 0, sizeof(SeFastpathPacketContextStruct)); frag->s = &core->fragment_state.s; memset(frag->s, 0, sizeof(SeFastpathPacketStateStruct)); /* Create a new Work Queue entry and then copy extra things in pc. */ if (cvmx_unlikely(octeon_se_fastpath_fragc_helper_alloc(fragc, fragc->pc, frag, len))) { OCTEON_SE_DEBUG(3, "Unable to create fragment\n"); return NULL; } /* For local reference. */ packet_out.u64 = frag->wqe->packet_ptr.u64; header = ((uint8_t *)cvmx_phys_to_ptr(packet_out.s.addr)) + frag->s->ip_offset; if (frag->s->ip_version_6) { uint8_t nh; OCTEON_SE_DEBUG(9, "Building IPv6 fragment\n"); /* Assert that headers fit into the first segment. */ OCTEON_SE_ASSERT(packet_out.s.size > (frag->s->ip_offset + OCTEON_SE_FASTPATH_IP6_HDRLEN + OCTEON_SE_FASTPATH_IP6_EXT_FRAG_HDRLEN)); memcpy(header, fragc->u.ipv6.frag_hdr, OCTEON_SE_FASTPATH_IP6_HDRLEN); OCTEON_SE_FASTPATH_IPH6_SET_LEN(header, len + OCTEON_SE_FASTPATH_IP6_EXT_FRAG_HDRLEN); OCTEON_SE_FASTPATH_IPH6_NH(header, nh); OCTEON_SE_FASTPATH_IPH6_SET_NH(header, OCTEON_SE_FASTPATH_IPPROTO_IPV6FRAG); /* Create the fragment header and copy it to its place. */ header += OCTEON_SE_FASTPATH_IP6_HDRLEN; header[0] = nh; header[SSH_IP6_EXT_FRAGMENT_OFS_RESERVED1] = 0; OCTEON_SE_PUT_16BIT(header + OCTEON_SE_FASTPATH_IP6_EXT_FRAGMENT_OFS_OFFSET, (fragc->offset | (is_last_frag ? 0 : 1))); OCTEON_SE_PUT_32BIT(header + OCTEON_SE_FASTPATH_IP6_EXT_FRAGMENT_OFS_ID, fragc->u.ipv6.id); /* Finally, copy the payload. */ octeon_se_fastpath_packet_buffer_create(fragment_buffer, packet_out, frag->s->ip_offset + OCTEON_SE_FASTPATH_IP6_HDRLEN + OCTEON_SE_FASTPATH_IP6_EXT_FRAG_HDRLEN, len, frag->wqe->word2.s.bufs); octeon_se_fastpath_buffer_copy(fragment_buffer, fragc->original_pkt, len); } else { /* Copy packet header to the fragment buffer. */ OCTEON_SE_DEBUG(9, "Build IPv4 fragment\n"); /* Asseet that header fits into the first segment. */ OCTEON_SE_ASSERT(packet_out.s.size > (frag->s->ip_offset + OCTEON_SE_FASTPATH_IP4_HDRLEN)); /* Copy in the IPv4 header first */ memcpy(header, fragc->u.ipv4.frag_hdr,OCTEON_SE_FASTPATH_IP4_HDRLEN); /* Copy data from the original packet to the fragment data part. */ octeon_se_fastpath_packet_buffer_create(fragment_buffer, packet_out, frag->s->ip_offset + OCTEON_SE_FASTPATH_IP4_HDRLEN, len, frag->wqe->word2.s.bufs); octeon_se_fastpath_buffer_copy(fragment_buffer, fragc->original_pkt, len); /* Compute new values for fragment offset and flag bits. */ OCTEON_SE_FASTPATH_IPH4_FRAG(header, fragoff_orig); offset_orig = (fragoff_orig & OCTEON_SE_FASTPATH_IP4_FRAG_MASK) << 3; fragoff = fragoff_orig & OCTEON_SE_FASTPATH_IPH4_FRAGOFF_RF; if (fragc->offset + data_len < fragc->total_len || (fragoff_orig & OCTEON_SE_FASTPATH_IPH4_FRAGOFF_MF)) fragoff |= OCTEON_SE_FASTPATH_IPH4_FRAGOFF_MF; /* If df_on_first_fragment is set and this is the first fragment, set DF bit */ if (fragc->offset == 0 && fragc->u.ipv4.df_on_first_fragment) fragoff |= OCTEON_SE_FASTPATH_IPH4_FRAGOFF_DF; OCTEON_SE_ASSERT((fragc->offset & 7) == 0); OCTEON_SE_FASTPATH_IPH4_SET_FRAG(header, (fragoff | ((fragc->offset + offset_orig) >> 3))); OCTEON_SE_FASTPATH_IPH4_SET_LEN(header, hlen + len); OCTEON_SE_FASTPATH_IPH4_SET_CHECKSUM(header, 0); checksum = octeon_se_fastpath_ip_cksum(header, hlen); OCTEON_SE_FASTPATH_IPH4_SET_CHECKSUM(header, checksum); } /* Update next fragment offset. */ fragc->offset += len; /* Return the fragment. */ return frag; }
/** * Enable an endpoint to respond to an IN transaction * * @param usb USB device state populated by * cvmx_usbd_initialize(). * @param endpoint_num * Endpoint number to enable * @param transfer_type * Transfer type for the endpoint * @param max_packet_size * Maximum packet size for the endpoint * @param buffer Buffer to send * @param buffer_length * Length of the buffer in bytes * * @return Zero on success, negative on failure */ int cvmx_usbd_in_endpoint_enable(cvmx_usbd_state_t *usb, int endpoint_num, cvmx_usbd_transfer_t transfer_type, int max_packet_size, uint64_t buffer, int buffer_length) { cvmx_usbcx_diepctlx_t usbc_diepctl; cvmx_usbcx_dieptsizx_t usbc_dieptsiz; if (cvmx_unlikely(usb->init_flags & CVMX_USBD_INITIALIZE_FLAGS_DEBUG)) cvmx_dprintf("%s: endpoint=%d buffer=0x%llx length=%d\n", __FUNCTION__, endpoint_num, (ULL)buffer, buffer_length); usb->endpoint[endpoint_num].buffer_length = buffer_length; CVMX_SYNCW; /* Flush out pending writes before enable */ /* Clear any pending interrupts */ __cvmx_usbd_write_csr32(usb, CVMX_USBCX_DIEPINTX(endpoint_num, usb->index), __cvmx_usbd_read_csr32(usb, CVMX_USBCX_DIEPINTX(endpoint_num, usb->index))); usbc_dieptsiz.u32 = 0; usbc_dieptsiz.s.mc = 1; if (buffer) { cvmx_write_csr(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + endpoint_num*8, buffer); usbc_dieptsiz.s.pktcnt = (buffer_length + max_packet_size - 1) / max_packet_size; if (usbc_dieptsiz.s.pktcnt == 0) usbc_dieptsiz.s.pktcnt = 1; usbc_dieptsiz.s.xfersize = buffer_length; } else { usbc_dieptsiz.s.pktcnt = 0; usbc_dieptsiz.s.xfersize = 0; } __cvmx_usbd_write_csr32(usb, CVMX_USBCX_DIEPTSIZX(endpoint_num, usb->index), usbc_dieptsiz.u32); usbc_diepctl.u32 = 0; usbc_diepctl.s.epena = (buffer != 0); usbc_diepctl.s.setd1pid = 0; usbc_diepctl.s.setd0pid = (buffer == 0); usbc_diepctl.s.cnak = 1; usbc_diepctl.s.txfnum = endpoint_num; usbc_diepctl.s.eptype = transfer_type; usbc_diepctl.s.usbactep = 1; usbc_diepctl.s.nextep = endpoint_num; if (endpoint_num == 0) { switch (max_packet_size) { case 8: usbc_diepctl.s.mps = 3; break; case 16: usbc_diepctl.s.mps = 2; break; case 32: usbc_diepctl.s.mps = 1; break; default: usbc_diepctl.s.mps = 0; break; } } else usbc_diepctl.s.mps = max_packet_size; __cvmx_usbd_write_csr32(usb, CVMX_USBCX_DIEPCTLX(endpoint_num, usb->index), usbc_diepctl.u32); return 0; }
/** * Poll the USB block for status and call all needed callback * handlers. This function is meant to be called in the interrupt * handler for the USB controller. It can also be called * periodically in a loop for non-interrupt based operation. * * @param usb USB device state populated by * cvmx_usbd_initialize(). * * @return Zero or negative on error. */ int cvmx_usbd_poll(cvmx_usbd_state_t *usb) { cvmx_usbcx_gintsts_t usbc_gintsts; if (cvmx_unlikely(usb->init_flags & CVMX_USBD_INITIALIZE_FLAGS_DEBUG)) cvmx_dprintf("%s: Called\n", __FUNCTION__); /* Read the pending interrupts */ usbc_gintsts.u32 = __cvmx_usbd_read_csr32(usb, CVMX_USBCX_GINTSTS(usb->index)); usbc_gintsts.u32 &= __cvmx_usbd_read_csr32(usb, CVMX_USBCX_GINTMSK(usb->index)); /* Clear the interrupts now that we know about them */ __cvmx_usbd_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index), usbc_gintsts.u32); if (usbc_gintsts.s.usbsusp) __cvmx_usbd_callback(usb, CVMX_USBD_CALLBACK_SUSPEND, 0, 0); if (usbc_gintsts.s.enumdone) __cvmx_usbd_callback(usb, CVMX_USBD_CALLBACK_ENUM_COMPLETE, 0, 0); if (usbc_gintsts.s.usbrst) { /* USB Reset (USBRst) The core sets this bit to indicate that a reset is detected on the USB. */ __cvmx_usbd_device_reset_complete(usb); __cvmx_usbd_callback(usb, CVMX_USBD_CALLBACK_RESET, 0, 0); } if (usbc_gintsts.s.oepint || usbc_gintsts.s.iepint) { cvmx_usbcx_daint_t usbc_daint; usbc_daint.u32 = __cvmx_usbd_read_csr32(usb, CVMX_USBCX_DAINT(usb->index)); if (usbc_daint.s.inepint) { int active_endpoints = usbc_daint.s.inepint; while (active_endpoints) { int endpoint; CVMX_CLZ(endpoint, active_endpoints); endpoint = 31 - endpoint; __cvmx_usbd_poll_in_endpoint(usb, endpoint); active_endpoints ^= 1<<endpoint; } } if (usbc_daint.s.outepint) { int active_endpoints = usbc_daint.s.outepint; while (active_endpoints) { int endpoint; CVMX_CLZ(endpoint, active_endpoints); endpoint = 31 - endpoint; __cvmx_usbd_poll_out_endpoint(usb, endpoint); active_endpoints ^= 1<<endpoint; } } } return 0; }
/** Execute outbound transforms */ SeFastpathRet octeon_se_fastpath_transform_out(SeFastpathCoreContext core, SeFastpath fastpath, SeFastpathPacketContext pc) { cvmx_buf_ptr_t packet_out; uint64_t packet_out_num_segs; size_t packet_out_len; SeFastpathTransformData se_trd; SeFastpathCombinedTransform combined; SeFastpathPacketBufferStruct src, dst; SeFastpathEspExtraInfoStruct extra_info[1]; SeFastpathMacExtraInfoStruct mac_info[1]; SeFastpathRet ret; uint8_t *header; uint32_t trd_i, tos, flow_label; uint64_t ipsec_seq; uint16_t csum, prefix_ofs; uint16_t esp_ah_ofs, prefix_len = 0, trailer_len = 0, pad_len = 0; uint8_t esp_ah_nh; uint64_t icv[OCTEON_SE_FASTPATH_MAX_HASH_WORDS] = { 0 }; size_t i; #ifdef OCTEON_SE_FASTPATH_TRANSFORM_AH size_t icv_pad_len = 0; #endif /* OCTEON_SE_FASTPATH_TRANSFORM_AH */ uint32_t run_time; size_t alignment = 0; #ifdef OCTEON_SE_FASTPATH_STATISTICS size_t out_octets; #endif /* OCTEON_SE_FASTPATH_STATISTICS */ OCTEON_SE_DEBUG(9, "Execute transform out\n"); packet_out.u64 = 0; OCTEON_SE_ASSERT(pc->transform_index != OCTEON_SE_FASTPATH_INVALID_INDEX); trd_i = pc->transform_index & 0x00ffffff; OCTEON_SE_ASSERT(trd_i < OCTEON_SE_FASTPATH_TRD_TABLE_SIZE); se_trd = OCTEON_SE_FASTPATH_TRD(fastpath, trd_i); OCTEON_SE_FASTPATH_TRD_READ_LOCK(fastpath, trd_i, se_trd); OCTEON_SE_FASTPATH_PREFETCH_TRD(se_trd); /* If transform is complex, pass packet to slowpath. */ if (cvmx_unlikely(se_trd->is_special)) { OCTEON_SE_DEBUG(9, "Special transform %08x, passing to slowpath\n", se_trd->transform); goto slowpath; } combined = octeon_se_fastpath_get_combined_transform(se_trd->transform, se_trd->mac_key_size); if (cvmx_unlikely(combined == NULL)) { OCTEON_SE_DEBUG(9, "Unsupported transform %08x, passing to slowpath\n", se_trd->transform); goto slowpath; } /* Update trd output timestamp. */ run_time = cvmx_fau_fetch_and_add32(OCTEON_SE_FASTPATH_FAU_RUNTIME, 0); cvmx_atomic_set32((int32_t *) &se_trd->last_out_packet_time, (int32_t) run_time); (*combined->init)(core->transform_context, se_trd->keymat + OCTEON_MAX_KEYMAT_LEN /2, se_trd->cipher_key_size, se_trd->keymat + OCTEON_MAX_KEYMAT_LEN /2 + OCTEON_MAX_ESP_KEY_BITS /8, se_trd->mac_key_size); prefix_ofs = pc->s->ip_offset; /* Check ttl. */ if (cvmx_unlikely(pc->s->ttl == 0)) { OCTEON_SE_DEBUG(3, "Zero TTL, dropping\n"); goto corrupt; } #ifdef OCTEON_SE_FASTPATH_TRANSFORM_TRANSPORT_MODE if (cvmx_unlikely(!se_trd->tunnel_mode)) { /* In transport mode insert the ESP/AH header between IP and transport headers. */ prefix_ofs += pc->s->tr_offset; esp_ah_nh = pc->s->ipproto; prefix_len = 0; } else #endif /* OCTEON_SE_FASTPATH_TRANSFORM_TRANSPORT_MODE */ { /* In tunnel mode insert IP and ESP/AH headers before IP header. */ if (se_trd->ip_version_6) prefix_len = OCTEON_SE_FASTPATH_IP6_HDRLEN; else prefix_len = OCTEON_SE_FASTPATH_IP4_HDRLEN; if (pc->s->ip_version_6) esp_ah_nh = OCTEON_SE_FASTPATH_IPPROTO_IPV6; else esp_ah_nh = OCTEON_SE_FASTPATH_IPPROTO_IPIP; } /* Calculate IPsec overhead. */ #ifdef OCTEON_SE_FASTPATH_TRANSFORM_NATT /* Reserve space for UDP NAT-T. */ if (se_trd->transform & OCTEON_SE_FASTPATH_IPSEC_NATT) prefix_len += OCTEON_SE_FASTPATH_UDP_HDRLEN; #endif /* OCTEON_SE_FASTPATH_TRANSFORM_NATT */ #ifdef OCTEON_SE_FASTPATH_TRANSFORM_AH if (cvmx_unlikely(se_trd->transform & OCTEON_SE_FASTPATH_IPSEC_AH)) { prefix_len += OCTEON_SE_FASTPATH_AH_HDRLEN + combined->icv_len; #ifdef OCTEON_SE_FASTPATH_TRANSFORM_SHA2 if (cvmx_unlikely((se_trd->ip_version_6 == 1) && (se_trd->transform & OCTEON_SE_FASTPATH_MAC_HMAC_SHA2)) ) { icv_pad_len = 4; prefix_len += 4; /* Align AH header to 64 bit boundary */ } #endif /* OCTEON_SE_FASTPATH_TRANSFORM_SHA2 */ trailer_len = 0; pad_len = 0; } else if (se_trd->transform & OCTEON_SE_FASTPATH_IPSEC_ESP) #endif /* OCTEON_SE_FASTPATH_TRANSFORM_AH */ { prefix_len += (OCTEON_SE_FASTPATH_ESP_HDRLEN + combined->cipher_iv_len); trailer_len = 2 + combined->icv_len; pad_len = (pc->s->ip_len + pc->s->ip_offset - prefix_ofs + 2) % combined->pad_boundary; if (pad_len != 0) pad_len = combined->pad_boundary - pad_len; } /* The actual length of the packet */ packet_out_len = pc->s->ip_len + prefix_len + pad_len + trailer_len; OCTEON_SE_DEBUG(9, "Resultant packet len is %d\n", (int) packet_out_len); /* Check result packet length. */ if (cvmx_unlikely(se_trd->pmtu_received && pc->mtu > se_trd->pmtu_received)) pc->mtu = se_trd->pmtu_received; ret = octeon_se_fastpath_transform_check_pmtu(pc, packet_out_len); if (cvmx_unlikely(ret == OCTEON_SE_FASTPATH_RET_DROP)) goto drop; else if (cvmx_unlikely(ret == OCTEON_SE_FASTPATH_RET_SLOWPATH)) goto slowpath; /* In tunnel mode decrement ttl of inner header. */ #ifdef OCTEON_SE_FASTPATH_TRANSFORM_TRANSPORT_MODE if (cvmx_likely(se_trd->tunnel_mode)) #endif /* OCTEON_SE_FASTPATH_TRANSFORM_TRANSPORT_MODE */ { header = cvmx_phys_to_ptr(pc->wqe->packet_ptr.s.addr) + pc->s->ip_offset; if (pc->s->ip_version_6) { /* Assert that header is in the first packet segment */ OCTEON_SE_ASSERT(pc->wqe->packet_ptr.s.size >= OCTEON_SE_FASTPATH_IP6_HDRLEN); OCTEON_SE_FASTPATH_IPH6_SET_HL(header, pc->s->ttl - 1); } else { /* Assert that header is in the first packet segment */ OCTEON_SE_ASSERT(pc->wqe->packet_ptr.s.size >= OCTEON_SE_FASTPATH_IP4_HDRLEN); OCTEON_SE_FASTPATH_IPH4_SET_TTL(header, pc->s->ttl - 1); OCTEON_SE_FASTPATH_IPH4_CHECKSUM(header, csum); csum = octeon_se_fastpath_csum_update_byte(csum, SSH_IPH4_OFS_TTL, pc->s->ttl, pc->s->ttl - 1); OCTEON_SE_FASTPATH_IPH4_SET_CHECKSUM(header, csum); } } /* Save df bit processing state */ pc->s->df_bit_processing = se_trd->df_bit_processing; /* Allocate packet buffer chain for result packet. Request that crypto result offset is 8 byte aligned. */ alignment = OCTEON_SE_ALIGN_64(prefix_ofs + prefix_len) - (prefix_ofs + prefix_len); packet_out.u64 = octeon_se_fastpath_alloc_packet_chain(packet_out_len + pc->s->ip_offset, alignment, &packet_out_num_segs); if (cvmx_unlikely(packet_out.u64 == 0)) { OCTEON_SE_DEBUG(3, "Result packet allocation failed\n"); goto drop; } #ifdef OCTEON_SE_FASTPATH_TRANSFORM_TRANSPORT_MODE /* In case of transport mode copy the l3 header.*/ if (cvmx_unlikely(prefix_ofs > pc->s->ip_offset)) { OCTEON_SE_DEBUG(9, "Copying headers to %p\n", cvmx_phys_to_ptr(packet_out.s.addr) + pc->s->ip_offset); /* Assert that l3 headers are in the first packet segment. */ OCTEON_SE_ASSERT(packet_out.s.size > prefix_ofs); memcpy(cvmx_phys_to_ptr(packet_out.s.addr) + pc->s->ip_offset, cvmx_phys_to_ptr(pc->wqe->packet_ptr.s.addr) + pc->s->ip_offset, prefix_ofs - pc->s->ip_offset); } #endif /* OCTEON_SE_FASTPATH_TRANSFORM_TRANSPORT_MODE */ /* Prepare Source buffer */ octeon_se_fastpath_packet_buffer_create(&src, pc->wqe->packet_ptr, prefix_ofs, pc->s->ip_len + pc->s->ip_offset - prefix_ofs, pc->wqe->word2.s.bufs); /* Count the number of bytes input to crypto processing. */ OCTEON_SE_FASTPATH_STATS(out_octets = pc->s->ip_len + pc->s->ip_offset - prefix_ofs); /* Build headers */ header = ((uint8_t *) cvmx_phys_to_ptr(packet_out.s.addr)) + prefix_ofs; /* Build outer header for tunnel mode and modify IP header for transport mode.*/ #ifdef OCTEON_SE_FASTPATH_TRANSFORM_TRANSPORT_MODE if (cvmx_unlikely(!se_trd->tunnel_mode && pc->s->ip_version_6 == 0)) { /* IPv4 transport mode. */ OCTEON_SE_DEBUG(9, "Modifying IPv4 header at %p\n", header); /* Modify original IPv4 header and change IP protocol and len. */ OCTEON_SE_FASTPATH_IPH4_SET_LEN(header, packet_out_len); OCTEON_SE_FASTPATH_IPH4_SET_PROTO(header, se_trd->nh); OCTEON_SE_FASTPATH_IPH4_CHECKSUM(header, csum); csum = octeon_se_fastpath_csum_update_byte(csum, OCTEON_SE_FASTPATH_IPH4_OFS_PROTO, pc->s->ipproto, se_trd->nh); csum = octeon_se_fastpath_csum_update_short(csum, OCTEON_SE_FASTPATH_IPH4_OFS_LEN, pc->s->ip_len, packet_out_len); OCTEON_SE_FASTPATH_IPH4_SET_CHECKSUM(header, csum); } else if (cvmx_unlikely(!se_trd->tunnel_mode && pc->s->ip_version_6 == 1)) { /* IPv6 transport mode. */ OCTEON_SE_DEBUG(9, "Modifying IPv6 header at %p\n", header); OCTEON_SE_FASTPATH_IPH6_SET_LEN(header, packet_out_len - OCTEON_SE_FASTPATH_IP6_HDRLEN); OCTEON_SE_FASTPATH_IPH6_SET_NH(header, se_trd->nh); } else #endif /* OCTEON_SE_FASTPATH_TRANSFORM_TRANSPORT_MODE */ if (se_trd->ip_version_6 == 0) { OCTEON_SE_ASSERT(se_trd->tunnel_mode); /* IPv4 tunnel mode. */ OCTEON_SE_DEBUG(9, "Building outer IPv4 header at %p\n", header); OCTEON_SE_ASSERT(packet_out.s.size > prefix_ofs + OCTEON_SE_FASTPATH_IP4_HDRLEN); OCTEON_SE_FASTPATH_IPH4_SET_VERSION(header, 4); OCTEON_SE_FASTPATH_IPH4_SET_HLEN(header, 5); tos = 0; OCTEON_SE_FASTPATH_IPH4_SET_TOS(header, tos); OCTEON_SE_FASTPATH_IPH4_SET_LEN(header, packet_out_len); if (pc->s->df_bit_processing == OCTEON_SE_FASTPATH_DF_CLEAR || (pc->s->df_bit_processing == OCTEON_SE_FASTPATH_DF_KEEP && pc->s->ipv4_df == 0)) { uint32_t id; OCTEON_SE_FASTPATH_GET_NEXT_IPV4_PACKET_ID(core, id); OCTEON_SE_FASTPATH_IPH4_SET_ID(header, id); OCTEON_SE_FASTPATH_IPH4_SET_FRAG(header, 0); pc->s->ipv4_df = 0; } else { OCTEON_SE_FASTPATH_IPH4_SET_ID(header, 0); OCTEON_SE_FASTPATH_IPH4_SET_FRAG(header, OCTEON_SE_FASTPATH_IPH4_FRAGOFF_DF); pc->s->ipv4_df = 1; } OCTEON_SE_FASTPATH_IPH4_SET_TTL(header, OCTEON_SE_FASTPATH_IP4_TUNNEL_MODE_TTL); OCTEON_SE_FASTPATH_IPH4_SET_PROTO(header, se_trd->nh); OCTEON_SE_FASTPATH_IPH4_SET_CHECKSUM(header, 0); OCTEON_SE_FASTPATH_IPH4_SET_SRC(header, se_trd->own_addr_low); OCTEON_SE_FASTPATH_IPH4_SET_DST(header, se_trd->gw_addr_low); csum = octeon_se_fastpath_ip_cksum(header, OCTEON_SE_FASTPATH_IP4_HDRLEN); OCTEON_SE_FASTPATH_IPH4_SET_CHECKSUM(header, csum); prefix_ofs += OCTEON_SE_FASTPATH_IP4_HDRLEN; } else if (se_trd->ip_version_6 == 1) { OCTEON_SE_ASSERT(se_trd->tunnel_mode); /* IPv6 tunnel mode. */ OCTEON_SE_DEBUG(9, "Building outer IPv6 header at %p\n", header); OCTEON_SE_FASTPATH_IPH6_SET_VERSION(header, 6); tos = 0; OCTEON_SE_FASTPATH_IPH6_SET_CLASS(header, tos); flow_label = 0; OCTEON_SE_FASTPATH_IPH6_SET_FLOW(header, flow_label); OCTEON_SE_FASTPATH_IPH6_SET_LEN(header, packet_out_len - OCTEON_SE_FASTPATH_IP6_HDRLEN); OCTEON_SE_FASTPATH_IPH6_SET_NH(header, se_trd->nh); OCTEON_SE_FASTPATH_IPH6_SET_HL(header, OCTEON_SE_FASTPATH_IP6_TUNNEL_MODE_HL); OCTEON_SE_FASTPATH_IPH6_SET_SRC_LOW(header, se_trd->own_addr_low); OCTEON_SE_FASTPATH_IPH6_SET_SRC_HIGH(header, se_trd->own_addr_high); OCTEON_SE_FASTPATH_IPH6_SET_DST_LOW(header, se_trd->gw_addr_low); OCTEON_SE_FASTPATH_IPH6_SET_DST_HIGH(header, se_trd->gw_addr_high); prefix_ofs += OCTEON_SE_FASTPATH_IP6_HDRLEN; } #ifdef OCTEON_SE_FASTPATH_TRANSFORM_NATT /* Should we add NATT header as well ? */ if (cvmx_unlikely(se_trd->transform & OCTEON_SE_FASTPATH_IPSEC_NATT)) { header = ((uint8_t *) cvmx_phys_to_ptr(packet_out.s.addr)) + prefix_ofs; OCTEON_SE_DEBUG(9, "Building UDP NAT-T header at %p\n", header); OCTEON_SE_ASSERT(packet_out.s.size > prefix_ofs + OCTEON_SE_FASTPATH_UDP_HDRLEN); OCTEON_SE_ASSERT((se_trd->transform & OCTEON_SE_FASTPATH_IPSEC_AH) == 0); OCTEON_SE_ASSERT(se_trd->nh == OCTEON_SE_FASTPATH_IPPROTO_UDP); OCTEON_SE_FASTPATH_UDPH_SET_SRCPORT(header, se_trd->natt_local_port); OCTEON_SE_FASTPATH_UDPH_SET_DSTPORT(header, se_trd->natt_remote_port); OCTEON_SE_FASTPATH_UDPH_SET_LEN(header, packet_out_len - (prefix_ofs - pc->s->ip_offset)); OCTEON_SE_FASTPATH_UDPH_SET_CHECKSUM(header, 0); prefix_ofs += OCTEON_SE_FASTPATH_UDP_HDRLEN; } #endif /* OCTEON_SE_FASTPATH_TRANSFORM_NATT */ /* Build ESP/AH */ esp_ah_ofs = prefix_ofs; header = ((uint8_t *) cvmx_phys_to_ptr(packet_out.s.addr)) + prefix_ofs; #ifdef OCTEON_SE_FASTPATH_TRANSFORM_AH if (se_trd->transform & OCTEON_SE_FASTPATH_IPSEC_AH) { uint32_t low_seq; OCTEON_SE_DEBUG(9, "Building AH header at %p\n", header); OCTEON_SE_ASSERT(packet_out.s.size > prefix_ofs + OCTEON_SE_FASTPATH_AH_HDRLEN + combined->icv_len + icv_pad_len); /* Get and increment next sequence atomically. Note that se_trd contains the last sequence number transmitted, thus sequence is incremented by one here. */ ipsec_seq = (uint64_t) cvmx_atomic_fetch_and_add64((int64_t *)&se_trd->seq, 1); ipsec_seq++; OCTEON_SE_FASTPATH_AHH_SET_NH(header, esp_ah_nh); OCTEON_SE_FASTPATH_AHH_SET_LEN(header, (combined->icv_len + icv_pad_len + 12) / 4 - 2); OCTEON_SE_FASTPATH_AHH_SET_RESERVED(header, 0); OCTEON_SE_FASTPATH_AHH_SET_SPI(header, se_trd->spi_out); CVMX_DEXT(low_seq, ipsec_seq, 0, 32); OCTEON_SE_FASTPATH_AHH_SET_SEQ(header, low_seq); prefix_ofs += OCTEON_SE_FASTPATH_AH_HDRLEN + combined->icv_len; /* ICV computation also needs ICV field initialized to zero. */ memcpy(mac_info->prefix.u8, header, OCTEON_SE_FASTPATH_AH_HDRLEN); memset(mac_info->prefix.u8 + OCTEON_SE_FASTPATH_AH_HDRLEN, 0, combined->icv_len); mac_info->prefix_len = OCTEON_SE_FASTPATH_AH_HDRLEN + combined->icv_len; #ifdef OCTEON_SE_FASTPATH_TRANSFORM_SHA2 if (cvmx_unlikely((se_trd->ip_version_6 == 1) && (se_trd->transform & OCTEON_SE_FASTPATH_MAC_HMAC_SHA2)) ) { prefix_ofs += 4; mac_info->prefix_len += 4; /* Use IPsec seq as AH padding for making 64 bit aligned. */ OCTEON_SE_PUT_32BIT_ALIGNED(mac_info->prefix.u8 + OCTEON_SE_FASTPATH_AH_HDRLEN + combined->icv_len, low_seq); } #endif /* OCTEON_SE_FASTPATH_TRANSFORM_SHA2 */ #ifdef OCTEON_SE_FASTPATH_TRANSFORM_LONGSEQ if (cvmx_unlikely(se_trd->transform & OCTEON_SE_FASTPATH_IPSEC_LONGSEQ)) { CVMX_DEXT(mac_info->suffix, ipsec_seq, 32, 32); mac_info->suffix_available = 1; } else #endif /* OCTEON_SE_FASTPATH_TRANSFORM_LONGSEQ */ mac_info->suffix_available = 0; /* Assert that crypto offset is 8 byte aligned */ OCTEON_SE_ASSERT(((uint64_t) (cvmx_phys_to_ptr(packet_out.s.addr) + prefix_ofs)) % 8 == 0); octeon_se_fastpath_packet_buffer_create(&dst, packet_out, prefix_ofs, packet_out_len + pc->s->ip_offset, packet_out_num_segs); if (se_trd->ip_version_6 == 1) octeon_se_fastpath_mac_add_ah_header6(packet_out, pc->s->ip_offset, combined->update, core->transform_context, 0); else octeon_se_fastpath_mac_add_ah_header4(packet_out, pc->s->ip_offset, combined->update, core->transform_context, 0); OCTEON_SE_DEBUG(9, "MAC prefix, len %d\n", mac_info->prefix_len); OCTEON_SE_HEXDUMP(9, mac_info->prefix.u8, mac_info->prefix_len); /* Do the actual transform */ (*combined->encrypt)(core->transform_context, &dst, &src, mac_info, NULL, icv); /* Copy ICV to packet. */ if (cvmx_likely(combined->icv_len % 4 == 0)) { for (i = 0; i < combined->icv_len; i += 4) { OCTEON_SE_PUT_32BIT_ALIGNED(cvmx_phys_to_ptr(packet_out.s.addr) + esp_ah_ofs + OCTEON_SE_FASTPATH_AH_HDRLEN + i, *(uint32_t *)(((uint8_t *)icv) + i)); } } else { memcpy(cvmx_phys_to_ptr(packet_out.s.addr) + esp_ah_ofs + OCTEON_SE_FASTPATH_AH_HDRLEN, icv, combined->icv_len); } #ifdef OCTEON_SE_FASTPATH_TRANSFORM_SHA2 if (cvmx_unlikely((se_trd->ip_version_6 == 1) && (se_trd->transform & OCTEON_SE_FASTPATH_MAC_HMAC_SHA2)) ) { /* Use IPsec seq as AH padding for making 64 bit aligned. */ OCTEON_SE_PUT_32BIT(cvmx_phys_to_ptr(packet_out.s.addr) + esp_ah_ofs + OCTEON_SE_FASTPATH_AH_HDRLEN + combined->icv_len, low_seq); } #endif /* OCTEON_SE_FASTPATH_TRANSFORM_SHA2 */ } else if (cvmx_likely(se_trd->transform & OCTEON_SE_FASTPATH_IPSEC_ESP)) #endif /* OCTEON_SE_FASTPATH_TRANSFORM_AH */ { uint32_t low_seq; OCTEON_SE_DEBUG(9, "Building ESP header at %p\n", header); /* Assert that there is enough space for ESP */ OCTEON_SE_ASSERT(packet_out.s.size > prefix_ofs + OCTEON_SE_FASTPATH_ESP_HDRLEN); /* Get and increment next sequence atomically. Note that se_trd contains the last sequence number transmitted, thus sequence is incremented by one here. */ ipsec_seq = (uint64_t) cvmx_atomic_fetch_and_add64((int64_t *)&se_trd->seq, 1); ipsec_seq++; /* Build ESP header. */ OCTEON_SE_FASTPATH_ESPH_SET_SPI(header, se_trd->spi_out); CVMX_DEXT(low_seq, ipsec_seq, 0, 32); OCTEON_SE_FASTPATH_ESPH_SET_SEQ(header, low_seq); prefix_ofs += OCTEON_SE_FASTPATH_ESP_HDRLEN; /* Fill in extra info for transform. */ extra_info->pad_len = pad_len; extra_info->nh = esp_ah_nh; /* Fill in extra data form MAC. */ OCTEON_SE_PUT_32BIT_ALIGNED(mac_info->prefix.u8, se_trd->spi_out); #ifdef OCTEON_SE_FASTPATH_TRANSFORM_AES_GCM if (cvmx_likely(combined->is_auth_cipher)) { /* Extract cipher nonce. */ OCTEON_SE_ASSERT(se_trd->cipher_nonce_size == 4); OCTEON_SE_GET_32BIT_ALIGNED(se_trd->keymat + OCTEON_MAX_KEYMAT_LEN /2 + se_trd->cipher_key_size, extra_info->cipher_nonce); /* Use IPsec seq# as counter. */ extra_info->iv[0] = ipsec_seq; #ifdef OCTEON_SE_FASTPATH_TRANSFORM_LONGSEQ if (cvmx_unlikely(se_trd->transform & OCTEON_SE_FASTPATH_IPSEC_LONGSEQ)) { OCTEON_SE_PUT_64BIT(&mac_info->prefix.u8[4], ipsec_seq); mac_info->prefix_len = 12; } else #endif /* OCTEON_SE_FASTPATH_TRANSFORM_LONGSEQ */ { OCTEON_SE_PUT_32BIT_ALIGNED(&mac_info->prefix.u8[4], low_seq); mac_info->prefix_len = 8; } } else #endif /* OCTEON_SE_FASTPATH_TRANSFORM_AES_GCM */ { for (i = 0; i < combined->cipher_iv_len / 8; i++) extra_info->iv[i] = cvmx_rng_get_random64(); /* Prepare extra mac information */ OCTEON_SE_PUT_32BIT_ALIGNED(&mac_info->prefix.u8[4], low_seq); mac_info->prefix_len = 8; #ifdef OCTEON_SE_FASTPATH_TRANSFORM_LONGSEQ if (cvmx_unlikely(se_trd->transform & OCTEON_SE_FASTPATH_IPSEC_LONGSEQ)) { CVMX_DEXT(mac_info->suffix, ipsec_seq, 32, 32); mac_info->suffix_available = 1; } else #endif /* OCTEON_SE_FASTPATH_TRANSFORM_LONGSEQ */ mac_info->suffix_available = 0; } /* Assert that crypto offset is 8 byte aligned */ OCTEON_SE_ASSERT(((uint64_t) (cvmx_phys_to_ptr(packet_out.s.addr) + prefix_ofs)) % 8 == 0); octeon_se_fastpath_packet_buffer_create(&dst, packet_out, prefix_ofs, packet_out_len + pc->s->ip_offset - prefix_ofs, packet_out_num_segs); OCTEON_SE_DEBUG(9, "Performing crypto transform\n"); /* Do the actual transform. */ (*combined->encrypt)(core->transform_context, &dst, &src, mac_info, extra_info, icv); /* The trailer should be appended at the end of encrypted data. Write ptr is pointing to correct location which may be unaligned if aes-gcm is used. */ OCTEON_SE_ASSERT(dst.total_bytes == combined->icv_len); OCTEON_SE_DEBUG(9, "Inserting ICV, len %d:\n", (int) combined->icv_len); OCTEON_SE_HEXDUMP(9, icv, combined->icv_len); octeon_se_fastpath_buffer_copy_in(&dst, icv, combined->icv_len); } /* Update trd statistics only after successful encryption. */ OCTEON_SE_FASTPATH_STATS({ cvmx_atomic_add64((int64_t *) &se_trd->out_octets, out_octets); cvmx_atomic_add64((int64_t *) &se_trd->out_packets, 1); });
/** * Initialize a USB port for use. This must be called before any * other access to the Octeon USB port is made. The port starts * off in the disabled state. * * @param usb Pointer to an empty cvmx_usbd_state_t structure * that will be populated by the initialize call. * This structure is then passed to all other USB * functions. * @param usb_port_number * Which Octeon USB port to initialize. * @param flags Flags to control hardware initialization. See * cvmx_usbd_initialize_flags_t for the flag * definitions. Some flags are mandatory. * * @return Zero or a negative on error. */ int cvmx_usbd_initialize(cvmx_usbd_state_t *usb, int usb_port_number, cvmx_usbd_initialize_flags_t flags) { cvmx_usbnx_clk_ctl_t usbn_clk_ctl; cvmx_usbnx_usbp_ctl_status_t usbn_usbp_ctl_status; if (cvmx_unlikely(flags & CVMX_USBD_INITIALIZE_FLAGS_DEBUG)) cvmx_dprintf("%s: Called\n", __FUNCTION__); memset(usb, 0, sizeof(*usb)); usb->init_flags = flags; usb->index = usb_port_number; /* Try to determine clock type automatically */ if ((usb->init_flags & (CVMX_USBD_INITIALIZE_FLAGS_CLOCK_XO_XI | CVMX_USBD_INITIALIZE_FLAGS_CLOCK_XO_GND)) == 0) { if (__cvmx_helper_board_usb_get_clock_type() == USB_CLOCK_TYPE_CRYSTAL_12) usb->init_flags |= CVMX_USBD_INITIALIZE_FLAGS_CLOCK_XO_XI; /* Only 12 MHZ crystals are supported */ else usb->init_flags |= CVMX_USBD_INITIALIZE_FLAGS_CLOCK_XO_GND; } if (usb->init_flags & CVMX_USBD_INITIALIZE_FLAGS_CLOCK_XO_GND) { /* Check for auto ref clock frequency */ if (!(usb->init_flags & CVMX_USBD_INITIALIZE_FLAGS_CLOCK_MHZ_MASK)) switch (__cvmx_helper_board_usb_get_clock_type()) { case USB_CLOCK_TYPE_REF_12: usb->init_flags |= CVMX_USBD_INITIALIZE_FLAGS_CLOCK_12MHZ; break; case USB_CLOCK_TYPE_REF_24: usb->init_flags |= CVMX_USBD_INITIALIZE_FLAGS_CLOCK_24MHZ; break; case USB_CLOCK_TYPE_REF_48: default: usb->init_flags |= CVMX_USBD_INITIALIZE_FLAGS_CLOCK_48MHZ; break; } } /* Power On Reset and PHY Initialization */ /* 1. Wait for DCOK to assert (nothing to do) */ /* 2a. Write USBN0/1_CLK_CTL[POR] = 1 and USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0 */ usbn_clk_ctl.u64 = cvmx_read_csr(CVMX_USBNX_CLK_CTL(usb->index)); usbn_clk_ctl.s.por = 1; usbn_clk_ctl.s.hrst = 0; usbn_clk_ctl.s.prst = 0; usbn_clk_ctl.s.hclk_rst = 0; usbn_clk_ctl.s.enable = 0; /* 2b. Select the USB reference clock/crystal parameters by writing appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON] */ if (usb->init_flags & CVMX_USBD_INITIALIZE_FLAGS_CLOCK_XO_GND) { /* The USB port uses 12/24/48MHz 2.5V board clock source at USB_XO. USB_XI should be tied to GND. Most Octeon evaluation boards require this setting */ if (OCTEON_IS_MODEL(OCTEON_CN3XXX)) { usbn_clk_ctl.cn31xx.p_rclk = 1; /* From CN31XX,CN30XX manual */ usbn_clk_ctl.cn31xx.p_xenbn = 0; } else if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN50XX)) usbn_clk_ctl.cn56xx.p_rtype = 2; /* From CN56XX,CN50XX manual */ else usbn_clk_ctl.cn52xx.p_rtype = 1; /* From CN52XX manual */ switch (usb->init_flags & CVMX_USBD_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) { case CVMX_USBD_INITIALIZE_FLAGS_CLOCK_12MHZ: usbn_clk_ctl.s.p_c_sel = 0; break; case CVMX_USBD_INITIALIZE_FLAGS_CLOCK_24MHZ: usbn_clk_ctl.s.p_c_sel = 1; break; case CVMX_USBD_INITIALIZE_FLAGS_CLOCK_48MHZ: usbn_clk_ctl.s.p_c_sel = 2; break; } } else { /* The USB port uses a 12MHz crystal as clock source at USB_XO and USB_XI */ if (OCTEON_IS_MODEL(OCTEON_CN3XXX)) { usbn_clk_ctl.cn31xx.p_rclk = 1; /* From CN31XX,CN30XX manual */ usbn_clk_ctl.cn31xx.p_xenbn = 1; } else if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN50XX)) usbn_clk_ctl.cn56xx.p_rtype = 0; /* From CN56XX,CN50XX manual */ else usbn_clk_ctl.cn52xx.p_rtype = 0; /* From CN52XX manual */ usbn_clk_ctl.s.p_c_sel = 0; } /* 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down such that USB is as close as possible to 125Mhz */ { int divisor = (cvmx_clock_get_rate(CVMX_CLOCK_CORE)+125000000-1)/125000000; if (divisor < 4) /* Lower than 4 doesn't seem to work properly */ divisor = 4; usbn_clk_ctl.s.divide = divisor; usbn_clk_ctl.s.divide2 = 0; } cvmx_write_csr(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64); /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */ usbn_clk_ctl.s.hclk_rst = 1; cvmx_write_csr(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64); /* 2e. Wait 64 core-clock cycles for HCLK to stabilize */ cvmx_wait(64); /* 3. Program the power-on reset field in the USBN clock-control register: USBN_CLK_CTL[POR] = 0 */ usbn_clk_ctl.s.por = 0; cvmx_write_csr(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64); /* 4. Wait 1 ms for PHY clock to start */ cvmx_wait_usec(1000); /* 5. Program the Reset input from automatic test equipment field in the USBP control and status register: USBN_USBP_CTL_STATUS[ATE_RESET] = 1 */ usbn_usbp_ctl_status.u64 = cvmx_read_csr(CVMX_USBNX_USBP_CTL_STATUS(usb->index)); usbn_usbp_ctl_status.s.ate_reset = 1; cvmx_write_csr(CVMX_USBNX_USBP_CTL_STATUS(usb->index), usbn_usbp_ctl_status.u64); /* 6. Wait 10 cycles */ cvmx_wait(10); /* 7. Clear ATE_RESET field in the USBN clock-control register: USBN_USBP_CTL_STATUS[ATE_RESET] = 0 */ usbn_usbp_ctl_status.s.ate_reset = 0; cvmx_write_csr(CVMX_USBNX_USBP_CTL_STATUS(usb->index), usbn_usbp_ctl_status.u64); /* 8. Program the PHY reset field in the USBN clock-control register: USBN_CLK_CTL[PRST] = 1 */ usbn_clk_ctl.s.prst = 1; cvmx_write_csr(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64); /* 9. Program the USBP control and status register to select host or device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for device */ usbn_usbp_ctl_status.s.hst_mode = 1; usbn_usbp_ctl_status.s.dm_pulld = 0; usbn_usbp_ctl_status.s.dp_pulld = 0; cvmx_write_csr(CVMX_USBNX_USBP_CTL_STATUS(usb->index), usbn_usbp_ctl_status.u64); /* 10. Wait 1 µs */ cvmx_wait_usec(1); /* 11. Program the hreset_n field in the USBN clock-control register: USBN_CLK_CTL[HRST] = 1 */ usbn_clk_ctl.s.hrst = 1; cvmx_write_csr(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64); /* 12. Proceed to USB core initialization */ usbn_clk_ctl.s.enable = 1; cvmx_write_csr(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64); cvmx_wait_usec(1); /* Program the following fields in the global AHB configuration register (USBC_GAHBCFG) DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode Burst length, USBC_GAHBCFG[HBSTLEN] = 0 Nonperiodic TxFIFO empty level (slave mode only), USBC_GAHBCFG[NPTXFEMPLVL] Periodic TxFIFO empty level (slave mode only), USBC_GAHBCFG[PTXFEMPLVL] Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1 */ { cvmx_usbcx_gahbcfg_t usbcx_gahbcfg; usbcx_gahbcfg.u32 = 0; usbcx_gahbcfg.s.dmaen = 1; usbcx_gahbcfg.s.hbstlen = 0; usbcx_gahbcfg.s.nptxfemplvl = 1; usbcx_gahbcfg.s.ptxfemplvl = 1; usbcx_gahbcfg.s.glblintrmsk = 1; __cvmx_usbd_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index), usbcx_gahbcfg.u32); } /* Program the following fields in USBC_GUSBCFG register. HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0 ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0 USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5 PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0 */ { cvmx_usbcx_gusbcfg_t usbcx_gusbcfg; usbcx_gusbcfg.u32 = __cvmx_usbd_read_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index)); usbcx_gusbcfg.s.toutcal = 0; usbcx_gusbcfg.s.ddrsel = 0; usbcx_gusbcfg.s.usbtrdtim = 0x5; usbcx_gusbcfg.s.phylpwrclksel = 0; __cvmx_usbd_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index), usbcx_gusbcfg.u32); } /* Program the following fields in the USBC0/1_DCFG register: Device speed, USBC0/1_DCFG[DEVSPD] = 0 (high speed) Non-zero-length status OUT handshake, USBC0/1_DCFG[NZSTSOUTHSHK]=0 Periodic frame interval (if periodic endpoints are supported), USBC0/1_DCFG[PERFRINT] = 1 */ { cvmx_usbcx_dcfg_t usbcx_dcfg; usbcx_dcfg.u32 = __cvmx_usbd_read_csr32(usb, CVMX_USBCX_DCFG(usb->index)); usbcx_dcfg.s.devspd = 0; usbcx_dcfg.s.nzstsouthshk = 0; usbcx_dcfg.s.perfrint = 1; __cvmx_usbd_write_csr32(usb, CVMX_USBCX_DCFG(usb->index), usbcx_dcfg.u32); } /* Program the USBC0/1_GINTMSK register */ { cvmx_usbcx_gintmsk_t usbcx_gintmsk; usbcx_gintmsk.u32 = __cvmx_usbd_read_csr32(usb, CVMX_USBCX_GINTMSK(usb->index)); usbcx_gintmsk.s.oepintmsk = 1; usbcx_gintmsk.s.inepintmsk = 1; usbcx_gintmsk.s.enumdonemsk = 1; usbcx_gintmsk.s.usbrstmsk = 1; usbcx_gintmsk.s.usbsuspmsk = 1; __cvmx_usbd_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index), usbcx_gintmsk.u32); } cvmx_usbd_disable(usb); return 0; }
/** * @INTERNAL * Perform USB device mode initialization after a reset completes. * This should be called after USBC0/1_GINTSTS[USBRESET] and * corresponds to section 22.6.1.1, "Initialization on USB Reset", * in the manual. * * @param usb USB device state populated by * cvmx_usbd_initialize(). * * @return Zero or negative on error. */ static int __cvmx_usbd_device_reset_complete(cvmx_usbd_state_t *usb) { cvmx_usbcx_ghwcfg2_t usbcx_ghwcfg2; cvmx_usbcx_ghwcfg3_t usbcx_ghwcfg3; cvmx_usbcx_doepmsk_t usbcx_doepmsk; cvmx_usbcx_diepmsk_t usbcx_diepmsk; cvmx_usbcx_daintmsk_t usbc_daintmsk; cvmx_usbcx_gnptxfsiz_t gnptxfsiz; int fifo_space; int i; if (cvmx_unlikely(usb->init_flags & CVMX_USBD_INITIALIZE_FLAGS_DEBUG)) cvmx_dprintf("%s: Processing reset\n", __FUNCTION__); usbcx_ghwcfg2.u32 = __cvmx_usbd_read_csr32(usb, CVMX_USBCX_GHWCFG2(usb->index)); usbcx_ghwcfg3.u32 = __cvmx_usbd_read_csr32(usb, CVMX_USBCX_GHWCFG3(usb->index)); /* Set up the data FIFO RAM for each of the FIFOs */ fifo_space = usbcx_ghwcfg3.s.dfifodepth; /* Start at the top of the FIFO and assign space for each periodic fifo */ for (i=usbcx_ghwcfg2.s.numdeveps; i>0; i--) { cvmx_usbcx_dptxfsizx_t siz; siz.u32 = __cvmx_usbd_read_csr32(usb, CVMX_USBCX_DPTXFSIZX(i, usb->index)); fifo_space -= siz.s.dptxfsize; siz.s.dptxfstaddr = fifo_space; __cvmx_usbd_write_csr32(usb, CVMX_USBCX_DPTXFSIZX(i, usb->index), siz.u32); } /* Assign half the leftover space to the non periodic tx fifo */ gnptxfsiz.u32 = __cvmx_usbd_read_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index)); gnptxfsiz.s.nptxfdep = fifo_space / 2; fifo_space -= gnptxfsiz.s.nptxfdep; gnptxfsiz.s.nptxfstaddr = fifo_space; __cvmx_usbd_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index), gnptxfsiz.u32); /* Assign the remain space to the RX fifo */ __cvmx_usbd_write_csr32(usb, CVMX_USBCX_GRXFSIZ(usb->index), fifo_space); /* Unmask the common endpoint interrupts */ usbcx_doepmsk.u32 = 0; usbcx_doepmsk.s.setupmsk = 1; usbcx_doepmsk.s.epdisbldmsk = 1; usbcx_doepmsk.s.xfercomplmsk = 1; __cvmx_usbd_write_csr32(usb, CVMX_USBCX_DOEPMSK(usb->index), usbcx_doepmsk.u32); usbcx_diepmsk.u32 = 0; usbcx_diepmsk.s.epdisbldmsk = 1; usbcx_diepmsk.s.xfercomplmsk = 1; __cvmx_usbd_write_csr32(usb, CVMX_USBCX_DIEPMSK(usb->index), usbcx_diepmsk.u32); usbc_daintmsk.u32 = 0; usbc_daintmsk.s.inepmsk = -1; usbc_daintmsk.s.outepmsk = -1; __cvmx_usbd_write_csr32(usb, CVMX_USBCX_DAINTMSK(usb->index), usbc_daintmsk.u32); /* Set all endpoints to NAK */ for (i=0; i<usbcx_ghwcfg2.s.numdeveps+1; i++) { cvmx_usbcx_doepctlx_t usbc_doepctl; usbc_doepctl.u32 = 0; usbc_doepctl.s.snak = 1; usbc_doepctl.s.usbactep = 1; usbc_doepctl.s.mps = (i==0) ? 0 : 64; __cvmx_usbd_write_csr32(usb, CVMX_USBCX_DOEPCTLX(i, usb->index), usbc_doepctl.u32); } return 0; }
/** * Process incoming packets. */ int inic_data_loop(void) { cvm_common_wqe_t *swp = NULL; cvm_tcp_in_endpoints_t conn; cvm_tcp_tcphdr_t *th = NULL; cvm_ip_ip_t *ih = NULL; cvmx_sysinfo_t *sys_info_ptr = cvmx_sysinfo_get(); uint64_t cpu_clock_hz = sys_info_ptr->cpu_clock_hz; uint64_t tick_cycle = cvmx_get_cycle(); uint64_t tick_step; uint32_t idle_processing_interval_ticks = (CVM_COMMON_IDLE_PROCESSING_INTERVAL)*(1000*1000)/(CVM_COMMON_TICK_LEN_US); uint32_t idle_processing_last_ticks = 0; #ifdef INET6 struct cvm_ip6_ip6_hdr *ip6 = NULL; #ifdef CVM_ENET_TUNNEL struct cvm_ip6_ip6_hdr *i6h = NULL; #endif #endif #ifdef CVM_CLI_APP uint64_t idle_cycle_start_value; #endif /* for the simulator */ if (cpu_clock_hz == 0) { cpu_clock_hz = 333000000; } tick_step = (CVM_COMMON_TICK_LEN_US * cpu_clock_hz) / 1000000; cvm_debug_print_interval = cpu_clock_hz; #ifndef REAL_HW /* for the simulator, set the debug interval to be 3M cycles */ cvm_debug_print_interval = 3000000; #endif #ifdef DUTY_CYCLE start_cycle = cvmx_get_cycle(); process_count = 0; #endif if (cvmx_coremask_first_core(coremask_data)) { /* Initiate a timer transaction for arp entry timeouts */ //if(cvm_enet_arp_timeout_init() != CVMX_TIM_STATUS_SUCCESS) //{ // printf("Failed init of cvm_ip_arp_timeout_init\n"); //} } #if defined(CVM_COMBINED_APP_STACK) /* Flush the packets sent by main_global and main_local */ /* printf("before cvm_send_packet () \n "); if (out_swp) { cvm_send_packet (); } printf("after cvm_send_packet () \n "); */ uint64_t app_timeout = cvmx_get_cycle (); #endif /* start the main loop */ while (1) { #ifdef DUTY_CYCLE end_cycle = cvmx_get_cycle(); /* check the wrap around case */ if (end_cycle < start_cycle) end_cycle += cpu_clock_hz; if ((end_cycle - start_cycle) > cvm_debug_print_interval) { inic_do_per_second_duty_cycle_processing(); } #endif /* DUTY_CYCLE */ cvmx_pow_work_request_async_nocheck(CVMX_SCR_WORK, 1); /* update the ticks variable */ while (cvmx_get_cycle() - tick_cycle > tick_step) { tick_cycle += tick_step; cvm_tcp_ticks++; if (!(cvm_tcp_ticks & 0x1f)) CVM_COMMON_HISTORY_SET_CYCLE(); } /* do common idle processing */ if ( (cvm_tcp_ticks - idle_processing_last_ticks) > idle_processing_interval_ticks) { if (cvmx_coremask_first_core(coremask_data)) { cvm_common_do_idle_processing(); } idle_processing_last_ticks = cvm_tcp_ticks; } #ifdef CVM_CLI_APP idle_cycle_start_value = cvmx_get_cycle(); #endif /* get work entry */ swp = (cvm_common_wqe_t *)cvmx_pow_work_response_async(CVMX_SCR_WORK); if (swp == NULL) { idle_counter++; if(core_id == highest_core_id) { cvm_enet_check_link_status(); } #ifdef CVM_CLI_APP cvmx_fau_atomic_add64(core_idle_cycles[core_id], (cvmx_get_cycle()-idle_cycle_start_value) ); #endif continue; } CVM_COMMON_EXTRA_STATS_ADD64 (CVM_FAU_REG_WQE_RCVD, 1); #ifdef WORK_QUEUE_ENTRY_SIZE_128 // { CVMX_PREFETCH0(swp); #else /* Prefetch work-queue entry */ CVMX_PREFETCH0(swp); CVMX_PREFETCH128(swp); #endif // WORK_QUEUE_ENTRY_SIZE_128 } out_swp = 0; out_swp_tail = 0; #ifdef DUTY_CYCLE /* we are about to start processing the packet - remember the cycle count */ process_start_cycle = cvmx_get_cycle(); #endif /* Short cut the common case */ if (cvmx_likely(swp->hw_wqe.unused == 0)) { goto packet_from_the_wire; } printf("Get work with unused is %X\n", swp->hw_wqe.unused); { { packet_from_the_wire: #if CVM_PKO_DONTFREE swp->hw_wqe.packet_ptr.s.i = 0; #endif #ifdef SANITY_CHECKS /* we have a work queue entry - do input sanity checks */ ret = cvm_common_input_sanity_and_buffer_count_update(swp); #endif if (cvmx_unlikely(swp->hw_wqe.word2.s.rcv_error)) { goto discard_swp; /* Receive error */ } #ifndef WORK_QUEUE_ENTRY_SIZE_128 // { { /* Make sure pre-fetch completed */ uint64_t dp = *(volatile uint64_t*)&swp->next; } #endif // WORK_QUEUE_ENTRY_SIZE_128 } { /* Initialize SW portion of the work-queue entry */ uint64_t *dptr = (uint64_t*)(&swp->next); dptr[0] = 0; dptr[1] = 0; dptr[2] = 0; dptr[3] = 0; } if(cvmx_unlikely(swp->hw_wqe.word2.s.not_IP)) { goto output; } /* Shortcut classification to avoid multiple lookups */ if( #ifndef INET6 swp->hw_wqe.word2.s.is_v6 || #endif swp->hw_wqe.word2.s.is_bcast #ifndef INET6 || swp->hw_wqe.word2.s.is_mcast #endif ) { goto discard_swp; /* Receive error */ } /* Packet is unicast IPv4, without L2 errors */ /* (All IP exceptions are dropped. This currently includes * IPv4 options and IPv6 extension headers.) */ if(cvmx_unlikely(swp->hw_wqe.word2.s.IP_exc)) { goto discard_swp; } /* Packet is Ipv4 (and no IP exceptions) */ if (cvmx_unlikely(swp->hw_wqe.word2.s.is_frag || !swp->hw_wqe.word2.s.tcp_or_udp)) { goto output; } #ifdef ANVL_RFC_793_COMPLIANCE /* RFC 793 says that: - We should send a RST out when we get a packet with FIN set without the ACK bit set in the flags field. - We should send a RST out when we get a packet with no flag set. Hence, let TCP stack handle these conditions. */ if (cvmx_unlikely(swp->hw_wqe.word2.s.L4_error && (cvmx_pip_l4_err_t)(swp->hw_wqe.word2.s.err_code != CVMX_PIP_TCP_FLG8_ERR) && (cvmx_pip_l4_err_t)(swp->hw_wqe.word2.s.err_code != CVMX_PIP_TCP_FLG9_ERR))) #else if (cvmx_unlikely(swp->hw_wqe.word2.s.L4_error)) #endif { cvm_tcp_handle_error(swp); goto discard_swp; } /* Packet is not fragmented, TCP/UDP, no IP exceptions/L4 errors */ /* We can try an L4 lookup now, but we need all the information */ ih = ((cvm_ip_ip_t *)&(swp->hw_wqe.packet_data[CVM_COMMON_PD_ALIGN])); if (!swp->hw_wqe.word2.s.is_v6) { /* for IPv4, we must subtract CVM_COMMON_PD_ALIGN rom tcp_offset to get the offset in the mbuf */ swp->l4_offset = ((uint16_t)(ih->ip_hl) << 2) + CVM_COMMON_PD_ALIGN; swp->l4_prot = ih->ip_p; } #ifdef INET6 else { ip6 = (struct cvm_ip6_ip6_hdr *) &swp->hw_wqe.packet_data[CVM_COMMON_IP6_PD_ALIGN]; CVM_COMMON_DBG_MSG (CVM_COMMON_DBG_LVL_5, "%s: %d Packet trace Src: %s/%d Dest: %s/%d prot: %d len: %d\n", __FUNCTION__, __LINE__, cvm_ip6_ip6_sprintf (&ip6->ip6_dst), conn.ie_fport, cvm_ip6_ip6_sprintf (&ip6->ip6_src), conn.ie_lport, swp->l4_prot, swp->hw_wqe.len); /* for IPv4, we must subtract CVM_COMMON_PD_ALIGN rom tcp_offset to get the offset in the mbuf */ swp->l4_offset = CVM_IP6_IP6_HDRLEN; swp->l4_prot = ip6->ip6_ctlun.ip6_un1.ip6_un1_nxt; } #endif th = ((cvm_tcp_tcphdr_t *)&(swp->hw_wqe.packet_data[swp->l4_offset])); /* check if it is a TCP packet */ if (swp->l4_prot == CVM_IP_IPPROTO_TCP) { process_handle(swp); #ifdef INET6 if (!swp->hw_wqe.word2.s.is_v6) #endif { CVM_TCP_TCP_DUMP ((void*)ih); /* assume IPv4 for now */ conn.ie_laddr = ih->ip_dst.s_addr; conn.ie_faddr = ih->ip_src.s_addr; conn.ie_lport = th->th_dport; conn.ie_fport = th->th_sport; } #ifdef INET6 else { /* assume IPv4 for now */ memcpy (&conn.ie6_laddr, &ip6->ip6_dst, sizeof (struct cvm_ip6_in6_addr)); memcpy (&conn.ie6_faddr, &ip6->ip6_src, sizeof (struct cvm_ip6_in6_addr)); conn.ie_lport = th->th_dport; conn.ie_fport = th->th_sport; /* do a TCP lookup */ swp->tcb = cvm_tcp6_lookup (swp); CVM_COMMON_DBG_MSG (CVM_COMMON_DBG_LVL_5, "%s: %d TCPv6 lookup Src: %s/%d Dest: %s/%d ret_tcb: 0x%llx\n", __FUNCTION__, __LINE__, cvm_ip6_ip6_sprintf ((cvm_ip6_in6_addr_t *) &conn.ie6_faddr), conn.ie_fport, cvm_ip6_ip6_sprintf ((cvm_ip6_in6_addr_t *) &conn.ie6_laddr), conn.ie_lport, CAST64(swp->tcb)); } #endif // INET6 } goto output; } /* packet from wire */ } /* switch */ output: CVMX_SYNCWS; /* Send packet out */ if (out_swp) { cvm_send_packet(); } if(swp != NULL) { S3_send_packet((cvmx_wqe_t *)swp); swp = NULL; } #ifdef DUTY_CYCLE process_end_cycle = cvmx_get_cycle(); process_count += (process_end_cycle - process_start_cycle); #endif } return (0); discard_swp: /* Free the chained buffers */ cvm_common_packet_free(swp); /* Free the work queue entry */ cvm_common_free_fpa_buffer(swp, CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE / CVMX_CACHE_LINE_SIZE); swp = NULL; goto output; } /* inic_data_loop */