/* Called to intialise thread-specific state, the first time a thread needs
* to use part of the per-thread state that requires explicit
* initialisation.
*
* Some members of oo_per_thread are implicitly initialised to zero either
* because they are static data (if HAVE_CC__THREAD), else when the memory
* is allocated. Those members must not be reinitialised here, because
* they may already have been used and modified.
*/
static void __oo_per_thread_init_thread(struct oo_per_thread* pt)
{
/* It's possible that we got here because we're not initialised at all! */
if( citp.init_level < CITP_INIT_SYSCALLS ) {
if( _citp_do_init_inprogress == 0 )
citp_do_init(CITP_INIT_ALL);
else
citp_do_init(CITP_INIT_SYSCALLS);
}
/* [pt->sig] is zero initialised. */
oo_stackname_thread_init(&pt->stackname);
pt->spinstate = 0;
#if CI_CFG_UDP
if( CITP_OPTS.udp_recv_spin )
pt->spinstate |= (1 << ONLOAD_SPIN_UDP_RECV);
if( CITP_OPTS.udp_send_spin )
pt->spinstate |= (1 << ONLOAD_SPIN_UDP_SEND);
#endif
if( CITP_OPTS.tcp_recv_spin )
pt->spinstate |= (1 << ONLOAD_SPIN_TCP_RECV);
if( CITP_OPTS.tcp_send_spin )
pt->spinstate |= (1 << ONLOAD_SPIN_TCP_SEND);
if( CITP_OPTS.tcp_accept_spin )
pt->spinstate |= (1 << ONLOAD_SPIN_TCP_ACCEPT);
if( CITP_OPTS.tcp_connect_spin )
pt->spinstate |= (1 << ONLOAD_SPIN_TCP_CONNECT);
if( CITP_OPTS.pkt_wait_spin )
pt->spinstate |= (1 << ONLOAD_SPIN_PKT_WAIT);
#if CI_CFG_USERSPACE_PIPE
if( CITP_OPTS.pipe_recv_spin )
pt->spinstate |= (1 << ONLOAD_SPIN_PIPE_RECV);
if( CITP_OPTS.pipe_send_spin )
pt->spinstate |= (1 << ONLOAD_SPIN_PIPE_SEND);
#endif
if( CITP_OPTS.ul_select_spin )
pt->spinstate |= (1 << ONLOAD_SPIN_SELECT);
if( CITP_OPTS.ul_poll_spin )
pt->spinstate |= (1 << ONLOAD_SPIN_POLL);
#if CI_CFG_USERSPACE_EPOLL
if( CITP_OPTS.ul_epoll_spin )
pt->spinstate |= (1 << ONLOAD_SPIN_EPOLL_WAIT);
#endif
if( CITP_OPTS.sock_lock_buzz )
pt->spinstate |= (1 << ONLOAD_SPIN_SOCK_LOCK);
if( CITP_OPTS.stack_lock_buzz )
pt->spinstate |= (1 << ONLOAD_SPIN_STACK_LOCK);
if( CITP_OPTS.so_busy_poll_spin )
pt->spinstate |= (1 << ONLOAD_SPIN_SO_BUSY_POLL);
}
void _init(void)
{
/* must not do any logging yet... */
if( citp_do_init(CITP_INIT_ALL) < 0 )
ci_fail(("EtherFabric transport library: failed to initialise (%d)",
citp.init_level));
Log_S(log("citp: initialisation done."));
}
/* Looks up the user-level 'FD info' for a given file descriptor.
** Returns pointer to the user-level 'FD info' for a given file
** descriptor, or NULL if the FD is not user-level.
** NOTE: The reference count of the 'FD info' is incremented, the
** caller should ensure the reference is dropped when no
** longer needed by calling citp_fdinfo_release_ref().
*/
citp_fdinfo* citp_fdtable_lookup_noprobe(unsigned fd)
{
/* Need to be initialised before we can try and grab the lock at the
** moment. TODO: make this more efficient by using a trylock to grab the
** fdtable lock, and on fail see if we need to initialise it.
*/
if( CI_UNLIKELY(citp.init_level < CITP_INIT_FDTABLE) ) {
if (_citp_do_init_inprogress == 0)
CI_TRY(citp_do_init(CITP_INIT_ALL));
else
CI_TRY(citp_do_init(CITP_INIT_FDTABLE)); /* get what we need */
return NULL;
}
if( fd < citp_fdtable.inited_count ) {
volatile citp_fdinfo_p* p_fdip = &citp_fdtable.table[fd].fdip;
citp_fdinfo_p fdip;
again:
/* Swap in the busy marker. */
fdip = *p_fdip;
if( fdip_is_normal(fdip) ) {
if( fdip_cas_succeed(p_fdip, fdip, fdip_busy) ) {
/* Bump the reference count. */
citp_fdinfo* fdi = fdip_to_fdi(fdip);
citp_fdinfo_ref(fdi);
/* Swap the busy marker out again. */
citp_fdtable_busy_clear(fd, fdip, 0);
return fdi;
}
goto again;
}
/* Not normal! */
else if( fdip_is_busy(fdip) ) {
citp_fdtable_busy_wait(fd, 0);
goto again;
}
}
return NULL;
}
void _init(void)
{
if (getpagesize() != CI_PAGE_SIZE)
ci_fail(("Page size mismatch, expected %u, "
"but the current value is %u",
CI_PAGE_SIZE, getpagesize()));
/* must not do any logging yet... */
if( citp_do_init(CITP_INIT_ALL) < 0 )
ci_fail(("EtherFabric transport library: failed to initialise (%d)",
citp.init_level));
Log_S(log("citp: initialisation done."));
}
citp_fdinfo*
citp_fdtable_lookup_fast(citp_lib_context_t* ctx, unsigned fd)
{
/* Note that if we haven't yet initialised this module, then
** [inited_count] will be zero, and the following test will fail. So the
** test for initialisation is done further down...
**
** This is highly performance critial. DO NOT add any code between here
** and the first [return] statement.
*/
citp_fdinfo* fdi;
/* Try to avoid entering lib. */
ctx->thread = NULL;
if( fd < citp_fdtable.inited_count ) {
volatile citp_fdinfo_p* p_fdip = &citp_fdtable.table[fd].fdip;
citp_fdinfo_p fdip;
again:
fdip = *p_fdip;
if( fdip_is_normal(fdip) ) {
citp_enter_lib_if(ctx);
if( citp_fdtable_is_mt_safe() ) {
/* No need to use atomic ops or add a ref to the fdi when MT-safe.
* The definition of "fds_mt_safe" is that the app does not change
* the meaning of a file descriptor in one thread when it is being
* used in another thread.
*/
fdi = fdip_to_fdi(fdip);
if( ! citp_fdinfo_is_consistent(fdi) )
fdi = citp_reprobe_moved(fdi, CI_TRUE, CI_FALSE);
return fdi;
}
else {
/* Swap in the busy marker. */
if( fdip_cas_succeed(p_fdip, fdip, fdip_busy) ) {
fdi = fdip_to_fdi(fdip);
ci_assert(fdi);
ci_assert_gt(oo_atomic_read(&fdi->ref_count), 0);
ci_assert(fdip_is_closing(fdip) || fdip_is_reserved(fdip) ||
fdi->fd == fd);
/* Bump the reference count. */
citp_fdinfo_ref(fdi);
if( ! citp_fdinfo_is_consistent(fdi) )
fdi = citp_reprobe_moved(fdi, CI_FALSE, CI_TRUE);
else {
/* Swap the busy marker out again. */
citp_fdtable_busy_clear(fd, fdip, 0);
}
return fdi;
}
goto again;
}
}
/* Not normal! */
if( fdip_is_passthru(fdip) )
return NULL;
citp_enter_lib_if(ctx);
if( fdip_is_busy(fdip) ) {
citp_fdtable_busy_wait(fd, 0);
goto again;
}
ci_assert(fdip_is_unknown(fdip));
goto probe;
}
if( citp.init_level < CITP_INIT_FDTABLE ) {
if( _citp_do_init_inprogress == 0 )
CI_TRY(citp_do_init(CITP_INIT_ALL));
else
CI_TRY(citp_do_init(CITP_INIT_FDTABLE)); /* get what we need */
}
if( fd >= citp_fdtable.size )
return NULL;
probe:
citp_enter_lib_if(ctx);
fdi = citp_fdtable_probe(fd);
if( fdi && citp_fdtable_is_mt_safe() )
citp_fdinfo_release_ref(fdi, 0);
return fdi;
}
citp_fdinfo *
citp_fdtable_lookup(unsigned fd)
{
/* Note that if we haven't yet initialised this module, then
** [inited_count] will be zero, and the following test will fail. So the
** test for initialisation is done further down...
**
** This is highly performance critial. DO NOT add any code between here
** and the first [return] statement.
*/
citp_fdinfo* fdi;
/* In some cases, we'll lock fdtable. Assert that it is possible: */
ci_assert(oo_per_thread_get()->sig.inside_lib);
if( fd < citp_fdtable.inited_count ) {
volatile citp_fdinfo_p* p_fdip = &citp_fdtable.table[fd].fdip;
citp_fdinfo_p fdip;
again:
/* Swap in the busy marker. */
fdip = *p_fdip;
if( fdip_is_normal(fdip) ) {
if( citp_fdtable_not_mt_safe() ) {
if( fdip_cas_succeed(p_fdip, fdip, fdip_busy) ) {
fdi = fdip_to_fdi(fdip);
ci_assert(fdi);
ci_assert_gt(oo_atomic_read(&fdi->ref_count), 0);
ci_assert(fdip_is_closing(fdip) || fdip_is_reserved(fdip) ||
fdi->fd == fd);
/* Bump the reference count. */
citp_fdinfo_ref(fdi);
if( ! citp_fdinfo_is_consistent(fdi) ) {
/* Something is wrong. Re-probe. */
fdi = citp_reprobe_moved(fdi, CI_FALSE, CI_TRUE);
}
else {
/* Swap the busy marker out again. */
citp_fdtable_busy_clear(fd, fdip, 0);
}
return fdi;
}
goto again;
}
else {
/* No need to use atomic ops when single-threaded. The definition
* of "fds_mt_safe" is that the app does not change the meaning of
* a file descriptor in one thread when it is being used in another
* thread. In that case I'm hoping this should be safe, but at
* time of writing I'm really not confident. (FIXME).
*/
fdi = fdip_to_fdi(fdip);
if( ci_is_multithreaded() )
citp_fdinfo_ref(fdi);
else
++fdi->ref_count.n;
if( ! citp_fdinfo_is_consistent(fdi) )
fdi = citp_reprobe_moved(fdi, CI_FALSE, CI_FALSE);
return fdi;
}
}
/* Not normal! */
if( fdip_is_passthru(fdip) ) return NULL;
if( fdip_is_busy(fdip) ) {
citp_fdtable_busy_wait(fd, 0);
goto again;
}
ci_assert(fdip_is_unknown(fdip));
goto probe;
}
if (citp.init_level < CITP_INIT_FDTABLE) {
if (_citp_do_init_inprogress == 0)
CI_TRY(citp_do_init(CITP_INIT_ALL));
else
CI_TRY(citp_do_init(CITP_INIT_FDTABLE)); /* get what we need */
}
if( fd >= citp_fdtable.size ) return NULL;
probe:
fdi = citp_fdtable_probe(fd);
return fdi;
}
