void nanotime(struct timespec *tsp) { struct bintime bt; bintime(&bt); bintime2timespec(&bt, tsp); }
void ffclock_getnanouptime(struct timespec *tsp) { struct bintime bt; ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST); bintime2timespec(&bt, tsp); }
void nanouptime(struct timespec *tsp) { struct bintime bt; nnanouptime++; binuptime(&bt); bintime2timespec(&bt, tsp); }
void getnanouptime(struct timespec *tsp) { struct timehands *th; u_int gen; do { th = timehands; gen = th->th_generation; bintime2timespec(&th->th_offset, tsp); } while (gen == 0 || gen != th->th_generation); }
int __vdso_clock_gettime(clockid_t clock_id, struct timespec *ts) { struct bintime bt; int abs, error; if (tk == NULL) { error = _elf_aux_info(AT_TIMEKEEP, &tk, sizeof(tk)); if (error != 0 || tk == NULL) return (ENOSYS); } if (tk->tk_ver != VDSO_TK_VER_CURR) return (ENOSYS); switch (clock_id) { case CLOCK_REALTIME: case CLOCK_REALTIME_PRECISE: case CLOCK_REALTIME_FAST: case CLOCK_SECOND: abs = 1; break; case CLOCK_MONOTONIC: case CLOCK_MONOTONIC_PRECISE: case CLOCK_MONOTONIC_FAST: case CLOCK_UPTIME: case CLOCK_UPTIME_PRECISE: case CLOCK_UPTIME_FAST: abs = 0; break; default: return (ENOSYS); } error = binuptime(&bt, tk, abs); if (error != 0) return (error); bintime2timespec(&bt, ts); if (clock_id == CLOCK_SECOND) ts->tv_nsec = 0; return (0); }
/* * Step our concept of UTC. This is done by modifying our estimate of * when we booted. * XXX: not locked. */ void tc_setclock(struct timespec *ts) { struct timespec ts2; struct bintime bt, bt2; binuptime(&bt2); timespec2bintime(ts, &bt); bintime_sub(&bt, &bt2); bintime_add(&bt2, &boottimebin); boottimebin = bt; bintime2timeval(&bt, &boottime); /* XXX fiddle all the little crinkly bits around the fiords... */ tc_windup(); if (timestepwarnings) { bintime2timespec(&bt2, &ts2); log(LOG_INFO, "Time stepped from %ld.%09ld to %ld.%09ld\n", (long)ts2.tv_sec, ts2.tv_nsec, (long)ts->tv_sec, ts->tv_nsec); } }
static void compute_stats(struct ctl_lun_io_stats *cur_stats, struct ctl_lun_io_stats *prev_stats, long double etime, long double *mbsec, long double *kb_per_transfer, long double *transfers_per_second, long double *ms_per_transfer, long double *ms_per_dma, long double *dmas_per_second) { uint64_t total_bytes = 0, total_operations = 0, total_dmas = 0; uint32_t port; struct bintime total_time_bt, total_dma_bt; struct timespec total_time_ts, total_dma_ts; int i; bzero(&total_time_bt, sizeof(total_time_bt)); bzero(&total_dma_bt, sizeof(total_dma_bt)); bzero(&total_time_ts, sizeof(total_time_ts)); bzero(&total_dma_ts, sizeof(total_dma_ts)); for (port = 0; port < CTL_MAX_PORTS; port++) { for (i = 0; i < CTL_STATS_NUM_TYPES; i++) { total_bytes += cur_stats->ports[port].bytes[i]; total_operations += cur_stats->ports[port].operations[i]; total_dmas += cur_stats->ports[port].num_dmas[i]; bintime_add(&total_time_bt, &cur_stats->ports[port].time[i]); bintime_add(&total_dma_bt, &cur_stats->ports[port].dma_time[i]); if (prev_stats != NULL) { total_bytes -= prev_stats->ports[port].bytes[i]; total_operations -= prev_stats->ports[port].operations[i]; total_dmas -= prev_stats->ports[port].num_dmas[i]; bintime_sub(&total_time_bt, &prev_stats->ports[port].time[i]); bintime_sub(&total_dma_bt, &prev_stats->ports[port].dma_time[i]); } } } *mbsec = total_bytes; *mbsec /= 1024 * 1024; if (etime > 0.0) *mbsec /= etime; else *mbsec = 0; *kb_per_transfer = total_bytes; *kb_per_transfer /= 1024; if (total_operations > 0) *kb_per_transfer /= total_operations; else *kb_per_transfer = 0; *transfers_per_second = total_operations; *dmas_per_second = total_dmas; if (etime > 0.0) { *transfers_per_second /= etime; *dmas_per_second /= etime; } else { *transfers_per_second = 0; *dmas_per_second = 0; } bintime2timespec(&total_time_bt, &total_time_ts); bintime2timespec(&total_dma_bt, &total_dma_ts); if (total_operations > 0) { /* * Convert the timespec to milliseconds. */ *ms_per_transfer = total_time_ts.tv_sec * 1000; *ms_per_transfer += total_time_ts.tv_nsec / 1000000; *ms_per_transfer /= total_operations; } else *ms_per_transfer = 0; if (total_dmas > 0) { /* * Convert the timespec to milliseconds. */ *ms_per_dma = total_dma_ts.tv_sec * 1000; *ms_per_dma += total_dma_ts.tv_nsec / 1000000; *ms_per_dma /= total_dmas; } else *ms_per_dma = 0; }
ssize_t netRecvEvent(Octet * buf, TimeInternal * time, NetPath * netPath, int flags) { ssize_t ret = 0; struct msghdr msg; struct iovec vec[1]; struct sockaddr_in from_addr; #ifdef PTPD_PCAP struct pcap_pkthdr *pkt_header; const u_char *pkt_data; #endif union { struct cmsghdr cm; char control[256]; } cmsg_un; struct cmsghdr *cmsg; #if defined(SO_TIMESTAMPNS) || defined(SO_TIMESTAMPING) struct timespec * ts; #elif defined(SO_BINTIME) struct bintime * bt; struct timespec ts; #endif #if defined(SO_TIMESTAMP) struct timeval * tv; #endif Boolean timestampValid = FALSE; #ifdef PTPD_PCAP if (netPath->pcapEvent == NULL) { /* Using sockets */ #endif vec[0].iov_base = buf; vec[0].iov_len = PACKET_SIZE; memset(&msg, 0, sizeof(msg)); memset(&from_addr, 0, sizeof(from_addr)); memset(buf, 0, PACKET_SIZE); memset(&cmsg_un, 0, sizeof(cmsg_un)); msg.msg_name = (caddr_t)&from_addr; msg.msg_namelen = sizeof(from_addr); msg.msg_iov = vec; msg.msg_iovlen = 1; msg.msg_control = cmsg_un.control; msg.msg_controllen = sizeof(cmsg_un.control); msg.msg_flags = 0; ret = recvmsg(netPath->eventSock, &msg, flags | MSG_DONTWAIT); if (ret <= 0) { if (errno == EAGAIN || errno == EINTR) return 0; return ret; }; if (msg.msg_flags & MSG_TRUNC) { ERROR("received truncated message\n"); return 0; } /* get time stamp of packet */ if (!time) { ERROR("null receive time stamp argument\n"); return 0; } if (msg.msg_flags & MSG_CTRUNC) { ERROR("received truncated ancillary data\n"); return 0; } #if defined(HAVE_DECL_MSG_ERRQUEUE) && HAVE_DECL_MSG_ERRQUEUE if(!(flags & MSG_ERRQUEUE)) #endif netPath->lastRecvAddr = from_addr.sin_addr.s_addr; netPath->receivedPackets++; if (msg.msg_controllen <= 0) { ERROR("received short ancillary data (%ld/%ld)\n", (long)msg.msg_controllen, (long)sizeof(cmsg_un.control)); return 0; } for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; cmsg = CMSG_NXTHDR(&msg, cmsg)) { if (cmsg->cmsg_level == SOL_SOCKET) { #if defined(SO_TIMESTAMPING) && defined(SO_TIMESTAMPNS) if(cmsg->cmsg_type == SO_TIMESTAMPING || cmsg->cmsg_type == SO_TIMESTAMPNS) { ts = (struct timespec *)CMSG_DATA(cmsg); time->seconds = ts->tv_sec; time->nanoseconds = ts->tv_nsec; timestampValid = TRUE; DBG("rcvevent: SO_TIMESTAMP%s %s time stamp: %us %dns\n", netPath->txTimestampFailure ? "NS" : "ING", (flags & MSG_ERRQUEUE) ? "(TX)" : "(RX)" , time->seconds, time->nanoseconds); break; } #elif defined(SO_TIMESTAMPNS) if(cmsg->cmsg_type == SCM_TIMESTAMPNS) { ts = (struct timespec *)CMSG_DATA(cmsg); time->seconds = ts->tv_sec; time->nanoseconds = ts->tv_nsec; timestampValid = TRUE; DBGV("kernel NANO recv time stamp %us %dns\n", time->seconds, time->nanoseconds); break; } #elif defined(SO_BINTIME) if(cmsg->cmsg_type == SCM_BINTIME) { bt = (struct bintime *)CMSG_DATA(cmsg); bintime2timespec(bt, &ts); time->seconds = ts.tv_sec; time->nanoseconds = ts.tv_nsec; timestampValid = TRUE; DBGV("kernel NANO recv time stamp %us %dns\n", time->seconds, time->nanoseconds); break; } #endif #if defined(SO_TIMESTAMP) if(cmsg->cmsg_type == SCM_TIMESTAMP) { tv = (struct timeval *)CMSG_DATA(cmsg); time->seconds = tv->tv_sec; time->nanoseconds = tv->tv_usec * 1000; timestampValid = TRUE; DBGV("kernel MICRO recv time stamp %us %dns\n", time->seconds, time->nanoseconds); } #endif } } if (!timestampValid) { /* * do not try to get by with recording the time here, better * to fail because the time recorded could be well after the * message receive, which would put a big spike in the * offset signal sent to the clock servo */ DBG("netRecvEvent: no receive time stamp\n"); return 0; } #ifdef PTPD_PCAP } #endif #ifdef PTPD_PCAP else { /* Using PCAP */ /* Discard packet on socket */ if (netPath->eventSock >= 0) { recv(netPath->eventSock, buf, PACKET_SIZE, MSG_DONTWAIT); } if ((ret = pcap_next_ex(netPath->pcapEvent, &pkt_header, &pkt_data)) < 1) { if (ret < 0) DBGV("netRecvEvent: pcap_next_ex failed %s\n", pcap_geterr(netPath->pcapEvent)); return 0; } /* Make sure this is IP (could dot1q get here?) */ if( ntohs(*(u_short *)(pkt_data + 12)) != ETHERTYPE_IP) DBGV("PCAP payload received is not Ethernet: 0x%04x\n", ntohs(*(u_short *)(pkt_data + 12))); /* Retrieve source IP from the payload - 14 eth + 12 IP */ netPath->lastRecvAddr = *(Integer32 *)(pkt_data + 26); netPath->receivedPackets++; /* XXX Total cheat */ memcpy(buf, pkt_data + netPath->headerOffset, pkt_header->caplen - netPath->headerOffset); time->seconds = pkt_header->ts.tv_sec; time->nanoseconds = pkt_header->ts.tv_usec * 1000; timestampValid = TRUE; DBGV("netRecvEvent: kernel PCAP recv time stamp %us %dns\n", time->seconds, time->nanoseconds); fflush(NULL); ret = pkt_header->caplen - netPath->headerOffset; } #endif return ret; }
ssize_t netRecvEvent(Octet * buf, TimeInternal * time, NetPath * netPath) { ssize_t ret; struct msghdr msg; struct iovec vec[1]; struct sockaddr_in from_addr; union { struct cmsghdr cm; char control[CMSG_SPACE(sizeof(struct timeval))]; } cmsg_un; struct cmsghdr *cmsg; #if defined(linux) struct timeval *tv; #else /* FreeBSD */ struct timespec ts; #endif /* FreeBSD or Linux */ vec[0].iov_base = buf; vec[0].iov_len = PACKET_SIZE; memset(&msg, 0, sizeof(msg)); memset(&from_addr, 0, sizeof(from_addr)); memset(buf, 0, PACKET_SIZE); memset(&cmsg_un, 0, sizeof(cmsg_un)); msg.msg_name = (caddr_t)&from_addr; msg.msg_namelen = sizeof(from_addr); msg.msg_iov = vec; msg.msg_iovlen = 1; msg.msg_control = cmsg_un.control; msg.msg_controllen = sizeof(cmsg_un.control); msg.msg_flags = 0; ret = recvmsg(netPath->eventSock, &msg, MSG_DONTWAIT); if (ret <= 0) { if (errno == EAGAIN || errno == EINTR) return 0; return ret; } if (msg.msg_flags & MSG_TRUNC) { ERROR("received truncated message\n"); return 0; } /* get time stamp of packet */ if (!time) { ERROR("null receive time stamp argument\n"); return 0; } if (msg.msg_flags & MSG_CTRUNC) { ERROR("received truncated ancillary data\n"); return 0; } if (msg.msg_controllen < sizeof(cmsg_un.control)) { ERROR("received short ancillary data (%d/%d)\n", msg.msg_controllen, (int)sizeof(cmsg_un.control)); return 0; } #if defined(linux) tv = 0; for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; cmsg = CMSG_NXTHDR(&msg, cmsg)) if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_TIMESTAMP) tv = (struct timeval *)CMSG_DATA(cmsg); if (tv) { time->seconds = tv->tv_sec; time->nanoseconds = tv->tv_usec * 1000; DBGV("kernel recv time stamp %us %dns\n", time->seconds, time->nanoseconds); } else { /* * do not try to get by with recording the time here, better * to fail because the time recorded could be well after the * message receive, which would put a big spike in the * offset signal sent to the clock servo */ DBG("no recieve time stamp\n"); return 0; } #else /* FreeBSD */ bzero(&ts, sizeof(ts)); for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; cmsg = CMSG_NXTHDR(&msg, cmsg)) if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_BINTIME) bintime2timespec((struct bintime *)CMSG_DATA(cmsg), &ts); if (ts.tv_sec != 0) { time->seconds = ts.tv_sec; time->nanoseconds = ts.tv_nsec; DBGV("kernel recv time stamp %us %dns\n", time->seconds, time->nanoseconds); } else { /* * do not try to get by with recording the time here, better * to fail because the time recorded could be well after the * message receive, which would put a big spike in the * offset signal sent to the clock servo */ DBG("no recieve time stamp\n"); return 0; } #endif /* FreeBSD or Linux */ return ret; }
/* * Initialize the next struct timehands in the ring and make * it the active timehands. Along the way we might switch to a different * timecounter and/or do seconds processing in NTP. Slightly magic. */ void tc_windup(void) { struct bintime bt; struct timehands *th, *tho; u_int64_t scale; u_int delta, ncount, ogen; int i; #ifdef leapsecs time_t t; #endif /* * Make the next timehands a copy of the current one, but do not * overwrite the generation or next pointer. While we update * the contents, the generation must be zero. */ tho = timehands; th = tho->th_next; ogen = th->th_generation; th->th_generation = 0; bcopy(tho, th, offsetof(struct timehands, th_generation)); /* * Capture a timecounter delta on the current timecounter and if * changing timecounters, a counter value from the new timecounter. * Update the offset fields accordingly. */ delta = tc_delta(th); if (th->th_counter != timecounter) ncount = timecounter->tc_get_timecount(timecounter); else ncount = 0; th->th_offset_count += delta; th->th_offset_count &= th->th_counter->tc_counter_mask; bintime_addx(&th->th_offset, th->th_scale * delta); #ifdef notyet /* * Hardware latching timecounters may not generate interrupts on * PPS events, so instead we poll them. There is a finite risk that * the hardware might capture a count which is later than the one we * got above, and therefore possibly in the next NTP second which might * have a different rate than the current NTP second. It doesn't * matter in practice. */ if (tho->th_counter->tc_poll_pps) tho->th_counter->tc_poll_pps(tho->th_counter); #endif /* * Deal with NTP second processing. The for loop normally * iterates at most once, but in extreme situations it might * keep NTP sane if timeouts are not run for several seconds. * At boot, the time step can be large when the TOD hardware * has been read, so on really large steps, we call * ntp_update_second only twice. We need to call it twice in * case we missed a leap second. */ bt = th->th_offset; bintime_add(&bt, &boottimebin); i = bt.sec - tho->th_microtime.tv_sec; if (i > LARGE_STEP) i = 2; for (; i > 0; i--) ntp_update_second(&th->th_adjustment, &bt.sec); /* Update the UTC timestamps used by the get*() functions. */ /* XXX shouldn't do this here. Should force non-`get' versions. */ bintime2timeval(&bt, &th->th_microtime); bintime2timespec(&bt, &th->th_nanotime); /* Now is a good time to change timecounters. */ if (th->th_counter != timecounter) { th->th_counter = timecounter; th->th_offset_count = ncount; } /*- * Recalculate the scaling factor. We want the number of 1/2^64 * fractions of a second per period of the hardware counter, taking * into account the th_adjustment factor which the NTP PLL/adjtime(2) * processing provides us with. * * The th_adjustment is nanoseconds per second with 32 bit binary * fraction and we want 64 bit binary fraction of second: * * x = a * 2^32 / 10^9 = a * 4.294967296 * * The range of th_adjustment is +/- 5000PPM so inside a 64bit int * we can only multiply by about 850 without overflowing, but that * leaves suitably precise fractions for multiply before divide. * * Divide before multiply with a fraction of 2199/512 results in a * systematic undercompensation of 10PPM of th_adjustment. On a * 5000PPM adjustment this is a 0.05PPM error. This is acceptable. * * We happily sacrifice the lowest of the 64 bits of our result * to the goddess of code clarity. * */ scale = (u_int64_t)1 << 63; scale += (th->th_adjustment / 1024) * 2199; scale /= th->th_counter->tc_frequency; th->th_scale = scale * 2; /* * Now that the struct timehands is again consistent, set the new * generation number, making sure to not make it zero. */ if (++ogen == 0) ogen = 1; th->th_generation = ogen; /* Go live with the new struct timehands. */ time_second = th->th_microtime.tv_sec; time_uptime = th->th_offset.sec; timehands = th; }
/* * Software (low priority) clock interrupt. * Run periodic events from timeout queue. */ void softclock(void *dummy) { struct callout *c; struct callout_tailq *bucket; int curticks; int steps; /* #steps since we last allowed interrupts */ int depth; int mpcalls; int mtxcalls; int gcalls; #ifdef DIAGNOSTIC struct bintime bt1, bt2; struct timespec ts2; static uint64_t maxdt = 36893488147419102LL; /* 2 msec */ static timeout_t *lastfunc; #endif #ifndef MAX_SOFTCLOCK_STEPS #define MAX_SOFTCLOCK_STEPS 100 /* Maximum allowed value of steps. */ #endif /* MAX_SOFTCLOCK_STEPS */ mpcalls = 0; mtxcalls = 0; gcalls = 0; depth = 0; steps = 0; mtx_lock_spin(&callout_lock); while (softticks != ticks) { softticks++; /* * softticks may be modified by hard clock, so cache * it while we work on a given bucket. */ curticks = softticks; bucket = &callwheel[curticks & callwheelmask]; c = TAILQ_FIRST(bucket); while (c) { depth++; if (c->c_time != curticks) { c = TAILQ_NEXT(c, c_links.tqe); ++steps; if (steps >= MAX_SOFTCLOCK_STEPS) { nextsoftcheck = c; /* Give interrupts a chance. */ mtx_unlock_spin(&callout_lock); ; /* nothing */ mtx_lock_spin(&callout_lock); c = nextsoftcheck; steps = 0; } } else { void (*c_func)(void *); void *c_arg; struct mtx *c_mtx; int c_flags; nextsoftcheck = TAILQ_NEXT(c, c_links.tqe); TAILQ_REMOVE(bucket, c, c_links.tqe); c_func = c->c_func; c_arg = c->c_arg; c_mtx = c->c_mtx; c_flags = c->c_flags; if (c->c_flags & CALLOUT_LOCAL_ALLOC) { c->c_func = NULL; c->c_flags = CALLOUT_LOCAL_ALLOC; SLIST_INSERT_HEAD(&callfree, c, c_links.sle); curr_callout = NULL; } else { c->c_flags = (c->c_flags & ~CALLOUT_PENDING); curr_callout = c; } curr_cancelled = 0; mtx_unlock_spin(&callout_lock); if (c_mtx != NULL) { mtx_lock(c_mtx); /* * The callout may have been cancelled * while we switched locks. */ if (curr_cancelled) { mtx_unlock(c_mtx); goto skip; } /* The callout cannot be stopped now. */ curr_cancelled = 1; if (c_mtx == &Giant) { gcalls++; CTR3(KTR_CALLOUT, "callout %p func %p arg %p", c, c_func, c_arg); } else { mtxcalls++; CTR3(KTR_CALLOUT, "callout mtx" " %p func %p arg %p", c, c_func, c_arg); } } else { mpcalls++; CTR3(KTR_CALLOUT, "callout mpsafe %p func %p arg %p", c, c_func, c_arg); } #ifdef DIAGNOSTIC binuptime(&bt1); #endif #ifdef MAXHE_TODO THREAD_NO_SLEEPING(); c_func(c_arg); THREAD_SLEEPING_OK(); #else c_func(c_arg); #endif // MAXHE_TODO #ifdef DIAGNOSTIC binuptime(&bt2); bintime_sub(&bt2, &bt1); if (bt2.frac > maxdt) { if (lastfunc != c_func || bt2.frac > maxdt * 2) { bintime2timespec(&bt2, &ts2); printf( "Expensive timeout(9) function: %p(%p) %jd.%09ld s\n", c_func, c_arg, (intmax_t)ts2.tv_sec, ts2.tv_nsec); } maxdt = bt2.frac; lastfunc = c_func; } #endif if ((c_flags & CALLOUT_RETURNUNLOCKED) == 0) mtx_unlock(c_mtx); skip: mtx_lock_spin(&callout_lock); curr_callout = NULL; if (callout_wait) { /* * There is someone waiting * for the callout to complete. */ callout_wait = 0; mtx_unlock_spin(&callout_lock); wakeup(&callout_wait); mtx_lock_spin(&callout_lock); } steps = 0; c = nextsoftcheck; } } } #ifdef MAXHE_TODO avg_depth += (depth * 1000 - avg_depth) >> 8; avg_mpcalls += (mpcalls * 1000 - avg_mpcalls) >> 8; avg_mtxcalls += (mtxcalls * 1000 - avg_mtxcalls) >> 8; avg_gcalls += (gcalls * 1000 - avg_gcalls) >> 8; #endif // MAXHE_TODO nextsoftcheck = NULL; mtx_unlock_spin(&callout_lock); }
static void softclock_call_cc(struct callout *c, struct callout_cpu *cc, int *mpcalls, int *lockcalls, int *gcalls) { void (*c_func)(void *); void *c_arg; int c_flags; #ifdef DIAGNOSTIC struct bintime bt1, bt2; struct timespec ts2; static uint64_t maxdt = 36893488147419102LL; /* 2 msec */ static timeout_t *lastfunc; #endif KASSERT((c->c_flags & (CALLOUT_PENDING | CALLOUT_ACTIVE)) == (CALLOUT_PENDING | CALLOUT_ACTIVE), ("softclock_call_cc: pend|act %p %x", c, c->c_flags)); c_func = c->c_func; c_arg = c->c_arg; c_flags = c->c_flags; if (c->c_flags & CALLOUT_LOCAL_ALLOC) c->c_flags = CALLOUT_LOCAL_ALLOC; else c->c_flags &= ~CALLOUT_PENDING; cc->cc_curr = c; cc->cc_cancel = 0; CC_UNLOCK(cc); { (*mpcalls)++; CTR3(KTR_CALLOUT, "callout mpsafe %p func %p arg %p", c, c_func, c_arg); } #ifdef DIAGNOSTIC binuptime(&bt1); #endif c_func(c_arg); #ifdef DIAGNOSTIC binuptime(&bt2); bintime_sub(&bt2, &bt1); if (bt2.frac > maxdt) { if (lastfunc != c_func || bt2.frac > maxdt * 2) { bintime2timespec(&bt2, &ts2); printf( "Expensive timeout(9) function: %p(%p) %jd.%09ld s\n", c_func, c_arg, (intmax_t)ts2.tv_sec, ts2.tv_nsec); } maxdt = bt2.frac; lastfunc = c_func; } #endif CTR1(KTR_CALLOUT, "callout %p finished", c); CC_LOCK(cc); KASSERT(cc->cc_curr == c, ("mishandled cc_curr")); cc->cc_curr = NULL; /* * If the current callout is locally allocated (from * timeout(9)) then put it on the freelist. * * Note: we need to check the cached copy of c_flags because * if it was not local, then it's not safe to deref the * callout pointer. */ KASSERT((c_flags & CALLOUT_LOCAL_ALLOC) == 0 || c->c_flags == CALLOUT_LOCAL_ALLOC, ("corrupted callout")); if (c_flags & CALLOUT_LOCAL_ALLOC) callout_cc_del(c, cc); }
/* * Software (low priority) clock interrupt. * Run periodic events from timeout queue. */ void softclock(void *dummy) { struct callout *c; struct callout_tailq *bucket; int curticks; int steps; /* #steps since we last allowed interrupts */ int depth; int mpcalls; int gcalls; int wakeup_cookie; #ifdef DIAGNOSTIC struct bintime bt1, bt2; struct timespec ts2; static uint64_t maxdt = 36893488147419102LL; /* 2 msec */ static timeout_t *lastfunc; #endif #ifndef MAX_SOFTCLOCK_STEPS #define MAX_SOFTCLOCK_STEPS 100 /* Maximum allowed value of steps. */ #endif /* MAX_SOFTCLOCK_STEPS */ mpcalls = 0; gcalls = 0; depth = 0; steps = 0; mtx_lock_spin(&callout_lock); while (softticks != ticks) { softticks++; /* * softticks may be modified by hard clock, so cache * it while we work on a given bucket. */ curticks = softticks; bucket = &callwheel[curticks & callwheelmask]; c = TAILQ_FIRST(bucket); while (c) { depth++; if (c->c_time != curticks) { c = TAILQ_NEXT(c, c_links.tqe); ++steps; if (steps >= MAX_SOFTCLOCK_STEPS) { nextsoftcheck = c; /* Give interrupts a chance. */ mtx_unlock_spin(&callout_lock); ; /* nothing */ mtx_lock_spin(&callout_lock); c = nextsoftcheck; steps = 0; } } else { void (*c_func)(void *); void *c_arg; int c_flags; nextsoftcheck = TAILQ_NEXT(c, c_links.tqe); TAILQ_REMOVE(bucket, c, c_links.tqe); c_func = c->c_func; c_arg = c->c_arg; c_flags = c->c_flags; c->c_func = NULL; if (c->c_flags & CALLOUT_LOCAL_ALLOC) { c->c_flags = CALLOUT_LOCAL_ALLOC; SLIST_INSERT_HEAD(&callfree, c, c_links.sle); } else { c->c_flags = (c->c_flags & ~CALLOUT_PENDING); } curr_callout = c; mtx_unlock_spin(&callout_lock); if (!(c_flags & CALLOUT_MPSAFE)) { mtx_lock(&Giant); gcalls++; CTR1(KTR_CALLOUT, "callout %p", c_func); } else { mpcalls++; CTR1(KTR_CALLOUT, "callout mpsafe %p", c_func); } #ifdef DIAGNOSTIC binuptime(&bt1); mtx_lock(&dont_sleep_in_callout); #endif c_func(c_arg); #ifdef DIAGNOSTIC mtx_unlock(&dont_sleep_in_callout); binuptime(&bt2); bintime_sub(&bt2, &bt1); if (bt2.frac > maxdt) { if (lastfunc != c_func || bt2.frac > maxdt * 2) { bintime2timespec(&bt2, &ts2); printf( "Expensive timeout(9) function: %p(%p) %jd.%09ld s\n", c_func, c_arg, (intmax_t)ts2.tv_sec, ts2.tv_nsec); } maxdt = bt2.frac; lastfunc = c_func; } #endif if (!(c_flags & CALLOUT_MPSAFE)) mtx_unlock(&Giant); mtx_lock_spin(&callout_lock); curr_callout = NULL; if (wakeup_needed) { /* * There might be someone waiting * for the callout to complete. */ wakeup_cookie = wakeup_ctr; mtx_unlock_spin(&callout_lock); mtx_lock(&callout_wait_lock); cv_broadcast(&callout_wait); wakeup_done_ctr = wakeup_cookie; mtx_unlock(&callout_wait_lock); mtx_lock_spin(&callout_lock); wakeup_needed = 0; } steps = 0; c = nextsoftcheck; } } } avg_depth += (depth * 1000 - avg_depth) >> 8; avg_mpcalls += (mpcalls * 1000 - avg_mpcalls) >> 8; avg_gcalls += (gcalls * 1000 - avg_gcalls) >> 8; nextsoftcheck = NULL; mtx_unlock_spin(&callout_lock); }
/* * Software (low priority) clock interrupt. * Run periodic events from timeout queue. */ void softclock(void *arg) { struct callout_cpu *cc; struct callout *c; struct callout_tailq *bucket; int curticks; int steps; /* #steps since we last allowed interrupts */ int depth; int mpcalls; int lockcalls; int gcalls; #ifdef DIAGNOSTIC struct bintime bt1, bt2; struct timespec ts2; static uint64_t maxdt = 36893488147419102LL; /* 2 msec */ static timeout_t *lastfunc; #endif #ifndef MAX_SOFTCLOCK_STEPS #define MAX_SOFTCLOCK_STEPS 100 /* Maximum allowed value of steps. */ #endif /* MAX_SOFTCLOCK_STEPS */ mpcalls = 0; lockcalls = 0; gcalls = 0; depth = 0; steps = 0; cc = (struct callout_cpu *)arg; CC_LOCK(cc); while (cc->cc_softticks != ticks) { /* * cc_softticks may be modified by hard clock, so cache * it while we work on a given bucket. */ curticks = cc->cc_softticks; cc->cc_softticks++; bucket = &cc->cc_callwheel[curticks & callwheelmask]; c = TAILQ_FIRST(bucket); while (c) { depth++; if (c->c_time != curticks) { c = TAILQ_NEXT(c, c_links.tqe); ++steps; if (steps >= MAX_SOFTCLOCK_STEPS) { cc->cc_next = c; /* Give interrupts a chance. */ CC_UNLOCK(cc); ; /* nothing */ CC_LOCK(cc); c = cc->cc_next; steps = 0; } } else { void (*c_func)(void *); void *c_arg; struct lock_class *class; struct lock_object *c_lock; int c_flags, sharedlock; cc->cc_next = TAILQ_NEXT(c, c_links.tqe); TAILQ_REMOVE(bucket, c, c_links.tqe); class = (c->c_lock != NULL) ? LOCK_CLASS(c->c_lock) : NULL; sharedlock = (c->c_flags & CALLOUT_SHAREDLOCK) ? 0 : 1; c_lock = c->c_lock; c_func = c->c_func; c_arg = c->c_arg; c_flags = c->c_flags; if (c->c_flags & CALLOUT_LOCAL_ALLOC) { c->c_flags = CALLOUT_LOCAL_ALLOC; } else { c->c_flags = (c->c_flags & ~CALLOUT_PENDING); } cc->cc_curr = c; cc->cc_cancel = 0; CC_UNLOCK(cc); if (c_lock != NULL) { class->lc_lock(c_lock, sharedlock); /* * The callout may have been cancelled * while we switched locks. */ if (cc->cc_cancel) { class->lc_unlock(c_lock); goto skip; } /* The callout cannot be stopped now. */ cc->cc_cancel = 1; if (c_lock == &Giant.lock_object) { gcalls++; CTR3(KTR_CALLOUT, "callout %p func %p arg %p", c, c_func, c_arg); } else { lockcalls++; CTR3(KTR_CALLOUT, "callout lock" " %p func %p arg %p", c, c_func, c_arg); } } else { mpcalls++; CTR3(KTR_CALLOUT, "callout mpsafe %p func %p arg %p", c, c_func, c_arg); } #ifdef DIAGNOSTIC binuptime(&bt1); #endif THREAD_NO_SLEEPING(); SDT_PROBE(callout_execute, kernel, , callout_start, c, 0, 0, 0, 0); c_func(c_arg); SDT_PROBE(callout_execute, kernel, , callout_end, c, 0, 0, 0, 0); THREAD_SLEEPING_OK(); #ifdef DIAGNOSTIC binuptime(&bt2); bintime_sub(&bt2, &bt1); if (bt2.frac > maxdt) { if (lastfunc != c_func || bt2.frac > maxdt * 2) { bintime2timespec(&bt2, &ts2); printf( "Expensive timeout(9) function: %p(%p) %jd.%09ld s\n", c_func, c_arg, (intmax_t)ts2.tv_sec, ts2.tv_nsec); } maxdt = bt2.frac; lastfunc = c_func; } #endif CTR1(KTR_CALLOUT, "callout %p finished", c); if ((c_flags & CALLOUT_RETURNUNLOCKED) == 0) class->lc_unlock(c_lock); skip: CC_LOCK(cc); /* * If the current callout is locally * allocated (from timeout(9)) * then put it on the freelist. * * Note: we need to check the cached * copy of c_flags because if it was not * local, then it's not safe to deref the * callout pointer. */ if (c_flags & CALLOUT_LOCAL_ALLOC) { KASSERT(c->c_flags == CALLOUT_LOCAL_ALLOC, ("corrupted callout")); c->c_func = NULL; SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle); } cc->cc_curr = NULL; if (cc->cc_waiting) { /* * There is someone waiting * for the callout to complete. */ cc->cc_waiting = 0; CC_UNLOCK(cc); wakeup(&cc->cc_waiting); CC_LOCK(cc); } steps = 0; c = cc->cc_next; } } }
void pps_event(struct pps_state *pps, int event) { struct bintime bt; struct timespec ts, *tsp, *osp; u_int tcount, *pcount; int foff, fhard; pps_seq_t *pseq; KASSERT(pps != NULL, ("NULL pps pointer in pps_event")); /* If the timecounter was wound up underneath us, bail out. */ if (pps->capgen == 0 || pps->capgen != pps->capth->th_generation) return; /* Things would be easier with arrays. */ if (event == PPS_CAPTUREASSERT) { tsp = &pps->ppsinfo.assert_timestamp; osp = &pps->ppsparam.assert_offset; foff = pps->ppsparam.mode & PPS_OFFSETASSERT; fhard = pps->kcmode & PPS_CAPTUREASSERT; pcount = &pps->ppscount[0]; pseq = &pps->ppsinfo.assert_sequence; } else { tsp = &pps->ppsinfo.clear_timestamp; osp = &pps->ppsparam.clear_offset; foff = pps->ppsparam.mode & PPS_OFFSETCLEAR; fhard = pps->kcmode & PPS_CAPTURECLEAR; pcount = &pps->ppscount[1]; pseq = &pps->ppsinfo.clear_sequence; } /* * If the timecounter changed, we cannot compare the count values, so * we have to drop the rest of the PPS-stuff until the next event. */ if (pps->ppstc != pps->capth->th_counter) { pps->ppstc = pps->capth->th_counter; *pcount = pps->capcount; pps->ppscount[2] = pps->capcount; return; } /* Convert the count to a timespec. */ tcount = pps->capcount - pps->capth->th_offset_count; tcount &= pps->capth->th_counter->tc_counter_mask; bt = pps->capth->th_offset; bintime_addx(&bt, pps->capth->th_scale * tcount); bintime_add(&bt, &boottimebin); bintime2timespec(&bt, &ts); /* If the timecounter was wound up underneath us, bail out. */ if (pps->capgen != pps->capth->th_generation) return; *pcount = pps->capcount; (*pseq)++; *tsp = ts; if (foff) { timespecadd(tsp, osp); if (tsp->tv_nsec < 0) { tsp->tv_nsec += 1000000000; tsp->tv_sec -= 1; } } #ifdef PPS_SYNC if (fhard) { uint64_t scale; /* * Feed the NTP PLL/FLL. * The FLL wants to know how many (hardware) nanoseconds * elapsed since the previous event. */ tcount = pps->capcount - pps->ppscount[2]; pps->ppscount[2] = pps->capcount; tcount &= pps->capth->th_counter->tc_counter_mask; scale = (uint64_t)1 << 63; scale /= pps->capth->th_counter->tc_frequency; scale *= 2; bt.sec = 0; bt.frac = 0; bintime_addx(&bt, scale * tcount); bintime2timespec(&bt, &ts); hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec); } #endif }