int wait_proper_time(struct timeval *start, int fps, unsigned int frame_num, int wait_flag) /* * wait_flag : 0 => return -1 or 0 * !0 => return -1, 0 or 1> * * return -1, 0 or over then 1 * -1 : no wait because too late * 0 : no wait, just in time * 1>: wait some second */ { struct timeval now; struct timeval elapse; struct timeval estimate; struct timeval now2; int diff_usec; int slp_c = 0; /* start of time check */ for (;;){ gettimeofday(&now, NULL); elapse = timeval_sub_timeval(&now, start); estimate.tv_sec = frame_num / fps; estimate.tv_usec = ((frame_num % fps)*1000*1000) / fps; diff_usec = timeval_diff_usec(&estimate, &elapse); if (!wait_flag){ break; } if (diff_usec <= 0){ break; } else { usleep(diff_usec); timeval_add_usec(&STAT.usleep_total, diff_usec); gettimeofday(&now2, NULL); diff_usec = timeval_diff_usec(&now2, &now); timeval_add_usec(&STAT.usleepr_total, diff_usec); d3_printf(" us:%d", diff_usec); slp_c++; /* break; */ /* check again! (=loop again!) */ /* to cape with too short usleep */ } } if (diff_usec < -(1000*1000/fps)) { /* too late, skip */ d3_printf("\nfram=%u: skip", frame_num); return -1; } d3_printf(" [%d]", slp_c); STAT.usleep_count += slp_c; return slp_c; }
static void __agent_schedule_abs(struct ice_agent *ag, const struct timeval *tv) { struct timeval nxt; long long diff; if (!ag) { ilog(LOG_ERR, "ice ag is NULL"); return; } nxt = *tv; mutex_lock(&ice_agents_timers_lock); if (ag->last_run.tv_sec) { /* make sure we don't run more often than we should */ diff = timeval_diff(&nxt, &ag->last_run); if (diff < TIMER_RUN_INTERVAL * 1000) timeval_add_usec(&nxt, TIMER_RUN_INTERVAL * 1000 - diff); } if (ag->next_check.tv_sec && timeval_cmp(&ag->next_check, &nxt) <= 0) goto nope; /* already scheduled sooner */ if (!g_tree_remove(ice_agents_timers, ag)) obj_hold(ag); /* if it wasn't removed, we make a new reference */ ag->next_check = nxt; g_tree_insert(ice_agents_timers, ag, ag); cond_broadcast(&ice_agents_timers_cond); nope: mutex_unlock(&ice_agents_timers_lock); }
static void __agent_schedule(struct ice_agent *ag, unsigned long usec) { struct timeval nxt; nxt = g_now; timeval_add_usec(&nxt, usec); __agent_schedule_abs(ag, &nxt); }
/* agent must NOT be locked, but call must be locked in R */ static void __do_ice_check(struct ice_candidate_pair *pair) { struct sockaddr_in6 dst; struct packet_stream *ps = pair->packet_stream; struct ice_agent *ag = pair->agent; u_int32_t prio, transact[3]; if (PAIR_ISSET(pair, SUCCEEDED) && !PAIR_ISSET(pair, TO_USE)) return; if (!ag->pwd[0].s) return; ZERO(dst); dst.sin6_port = htons(pair->remote_candidate->endpoint.port); dst.sin6_addr = pair->remote_candidate->endpoint.ip46; dst.sin6_family = AF_INET6; prio = ice_priority(ICT_PRFLX, pair->local_address->preference, pair->remote_candidate->component_id); mutex_lock(&ag->lock); pair->retransmit = g_now; if (!PAIR_SET(pair, IN_PROGRESS)) { PAIR_CLEAR2(pair, FROZEN, FAILED); pair->retransmit_ms = STUN_RETRANSMIT_INTERVAL; pair->retransmits = 0; } else if (pair->retransmits > STUN_MAX_RETRANSMITS) { __fail_pair(pair); mutex_unlock(&ag->lock); return; } else { pair->retransmit_ms *= 2; pair->retransmits++; } timeval_add_usec(&pair->retransmit, pair->retransmit_ms * 1000); __agent_schedule_abs(pair->agent, &pair->retransmit); memcpy(transact, pair->stun_transaction, sizeof(transact)); pair->was_controlling = AGENT_ISSET(ag, CONTROLLING); pair->was_nominated = PAIR_ISSET(pair, TO_USE); mutex_unlock(&ag->lock); ilog(LOG_DEBUG, "Sending %sICE/STUN request for candidate pair "PAIR_FORMAT" from %s to %s", PAIR_ISSET(pair, TO_USE) ? "nominating " : "", PAIR_FMT(pair), smart_ntop_buf(&pair->local_address->addr), smart_ntop_ep_buf(&pair->remote_candidate->endpoint)); stun_binding_request(&dst, transact, &ag->pwd[0], ag->ufrag, AGENT_ISSET(ag, CONTROLLING), tie_breaker, prio, &pair->local_address->addr, ps->sfd->fd.fd, PAIR_ISSET(pair, TO_USE)); }
void ice_thread_run(void *p) { struct ice_agent *ag; struct call *call; long long sleeptime; struct timeval tv; mutex_lock(&ice_agents_timers_lock); while (!g_shutdown) { gettimeofday(&g_now, NULL); /* lock our list and get the first element */ ag = g_tree_find_first(ice_agents_timers, NULL, NULL); /* scheduled to run? if not, we just go to sleep, otherwise we remove it from the tree, * steal the reference and run it */ if (!ag) goto sleep; if (timeval_cmp(&g_now, &ag->next_check) < 0) goto sleep; g_tree_remove(ice_agents_timers, ag); ZERO(ag->next_check); ag->last_run = g_now; mutex_unlock(&ice_agents_timers_lock); /* this agent is scheduled to run right now */ /* lock the call */ call = ag->call; log_info_ice_agent(ag); rwlock_lock_r(&call->master_lock); /* and run our checks */ __do_ice_checks(ag); /* finally, release our reference and start over */ log_info_clear(); rwlock_unlock_r(&call->master_lock); obj_put(ag); mutex_lock(&ice_agents_timers_lock); continue; sleep: /* figure out how long we should sleep */ sleeptime = ag ? timeval_diff(&ag->next_check, &g_now) : 100000; sleeptime = MIN(100000, sleeptime); /* 100 ms at the most */ tv = g_now; timeval_add_usec(&tv, sleeptime); cond_timedwait(&ice_agents_timers_cond, &ice_agents_timers_lock, &tv); continue; } mutex_unlock(&ice_agents_timers_lock); }
/* agent must NOT be locked, but call must be locked in R */ static void __do_ice_check(struct ice_candidate_pair *pair) { struct stream_fd *sfd = pair->sfd; struct ice_agent *ag = pair->agent; u_int32_t prio, transact[3]; if (PAIR_ISSET(pair, SUCCEEDED) && !PAIR_ISSET(pair, TO_USE)) return; if (!ag->pwd[0].s) return; prio = ice_priority(ICT_PRFLX, pair->local_intf->unique_id, pair->remote_candidate->component_id); mutex_lock(&ag->lock); pair->retransmit = g_now; if (!PAIR_SET(pair, IN_PROGRESS)) { PAIR_CLEAR2(pair, FROZEN, FAILED); pair->retransmit_ms = STUN_RETRANSMIT_INTERVAL; pair->retransmits = 0; } else if (pair->retransmits > STUN_MAX_RETRANSMITS) { __fail_pair(pair); mutex_unlock(&ag->lock); return; } else { pair->retransmit_ms *= 2; pair->retransmits++; } timeval_add_usec(&pair->retransmit, pair->retransmit_ms * 1000); __agent_schedule_abs(pair->agent, &pair->retransmit); memcpy(transact, pair->stun_transaction, sizeof(transact)); pair->was_controlling = AGENT_ISSET(ag, CONTROLLING); pair->was_nominated = PAIR_ISSET(pair, TO_USE); mutex_unlock(&ag->lock); ilog(LOG_DEBUG, "Sending %sICE/STUN request for candidate pair "PAIR_FORMAT" from %s to %s", PAIR_ISSET(pair, TO_USE) ? "nominating " : "", PAIR_FMT(pair), sockaddr_print_buf(&pair->local_intf->spec->local_address.addr), endpoint_print_buf(&pair->remote_candidate->endpoint)); stun_binding_request(&pair->remote_candidate->endpoint, transact, &ag->pwd[0], ag->ufrag, AGENT_ISSET(ag, CONTROLLING), tie_breaker, prio, &sfd->socket, PAIR_ISSET(pair, TO_USE)); }
void timerthread_run(void *p) { struct timerthread *tt = p; mutex_lock(&tt->lock); while (!rtpe_shutdown) { gettimeofday(&rtpe_now, NULL); /* lock our list and get the first element */ struct timerthread_obj *tt_obj = g_tree_find_first(tt->tree, NULL, NULL); /* scheduled to run? if not, we just go to sleep, otherwise we remove it from the tree, * steal the reference and run it */ if (!tt_obj) goto sleep; if (timeval_cmp(&rtpe_now, &tt_obj->next_check) < 0) goto sleep; // steal reference g_tree_remove(tt->tree, tt_obj); ZERO(tt_obj->next_check); tt_obj->last_run = rtpe_now; mutex_unlock(&tt->lock); // run and release tt->func(tt_obj); obj_put(tt_obj); mutex_lock(&tt->lock); continue; sleep:; /* figure out how long we should sleep */ long long sleeptime = tt_obj ? timeval_diff(&tt_obj->next_check, &rtpe_now) : 100000; sleeptime = MIN(100000, sleeptime); /* 100 ms at the most */ struct timeval tv = rtpe_now; timeval_add_usec(&tv, sleeptime); cond_timedwait(&tt->cond, &tt->lock, &tv); } mutex_unlock(&tt->lock); }
/* call is locked in R */ int ice_response(struct packet_stream *ps, struct sockaddr_in6 *src, struct in6_addr *dst, struct stun_attrs *attrs, u_int32_t transaction[3]) { struct ice_candidate_pair *pair, *opair; struct ice_agent *ag; struct call_media *media = ps->media; const char *err; unsigned int component; struct ice_candidate *cand; struct interface_address *ifa; int ret, was_ctl; __DBG("received ICE response from %s on %s", smart_ntop_port_buf(src), smart_ntop_buf(dst)); ag = media->ice_agent; if (!ag) return -1; atomic64_set(&ag->last_activity, poller_now); mutex_lock(&ag->lock); pair = g_hash_table_lookup(ag->transaction_hash, transaction); err = "ICE/STUN response with unknown transaction received"; if (!pair) goto err_unlock; was_ctl = pair->was_controlling; mutex_unlock(&ag->lock); ifa = pair->local_address; ilog(LOG_DEBUG, "Received ICE/STUN response code %u for candidate pair "PAIR_FORMAT" from %s to %s", attrs->error_code, PAIR_FMT(pair), smart_ntop_ep_buf(&pair->remote_candidate->endpoint), smart_ntop_buf(&ifa->addr)); /* verify endpoints */ err = "ICE/STUN response received, but source address didn't match remote candidate address"; if (memcmp(&src->sin6_addr, &pair->remote_candidate->endpoint.ip46, sizeof(src->sin6_addr))) goto err; if (ntohs(src->sin6_port) != pair->remote_candidate->endpoint.port) goto err; err = "ICE/STUN response received, but destination address didn't match local interface address"; if (memcmp(dst, &ifa->addr, sizeof(*dst))) goto err; if (pair->packet_stream != ps) goto err; PAIR_CLEAR(pair, IN_PROGRESS); ret = 0; /* handle all errors */ if (attrs->error_code) { err = "ICE/STUN error received"; if (attrs->error_code != 487) goto err; __role_change(ag, !was_ctl); __trigger_check(pair); goto out; } /* we don't discover peer reflexive here (RFC 5245 7.1.3.2.1) as we don't expect to be behind NAT */ /* we also skip parts of 7.1.3.2.2 as we don't do server reflexive */ mutex_lock(&ag->lock); /* check if we're in the final (controlling) phase */ if (pair->was_nominated && PAIR_CLEAR(pair, TO_USE)) { ilog(LOG_DEBUG, "Setting nominated ICE candidate pair "PAIR_FORMAT" as valid", PAIR_FMT(pair)); PAIR_SET(pair, VALID); g_tree_insert(ag->valid_pairs, pair, pair); ret = __check_valid(ag); goto out_unlock; } if (PAIR_SET(pair, SUCCEEDED)) goto out_unlock; ilog(LOG_DEBUG, "Setting ICE candidate pair "PAIR_FORMAT" as succeeded", PAIR_FMT(pair)); g_tree_insert(ag->succeeded_pairs, pair, pair); if (!ag->start_nominating.tv_sec) { if (__check_succeeded_complete(ag)) { ag->start_nominating = g_now; timeval_add_usec(&ag->start_nominating, 100000); __agent_schedule_abs(ag, &ag->start_nominating); } } /* now unfreeze all other pairs from the same foundation */ for (component = 1; component <= MAX_COMPONENTS; component++) { if (component == ps->component) continue; cand = __foundation_lookup(ag, &pair->remote_candidate->foundation, component); if (!cand) continue; opair = __pair_lookup(ag, cand, ifa); if (!opair) continue; if (PAIR_ISSET(opair, FAILED)) continue; if (!PAIR_CLEAR(opair, FROZEN)) continue; ilog(LOG_DEBUG, "Unfreezing related ICE pair "PAIR_FORMAT, PAIR_FMT(opair)); } /* if this was previously nominated by the peer, it's now valid */ if (PAIR_ISSET(pair, NOMINATED)) { PAIR_SET(pair, VALID); g_tree_insert(ag->valid_pairs, pair, pair); if (!AGENT_ISSET(ag, CONTROLLING)) ret = __check_valid(ag); } out_unlock: mutex_unlock(&ag->lock); out: return ret; err_unlock: mutex_unlock(&ag->lock); err: if (err) ilog(LOG_NOTICE, "%s (from %s on interface %s)", err, smart_ntop_port_buf(src), smart_ntop_buf(dst)); if (pair && attrs->error_code) __fail_pair(pair); return 0; }
static int __ensure_codec_handler(struct media_player *mp, AVStream *avs) { if (mp->handler) return 0; // synthesise rtp payload type struct rtp_payload_type src_pt = { .payload_type = -1 }; // src_pt.codec_def = codec_find_by_av(avs->codec->codec_id); `codec` is deprecated src_pt.codec_def = codec_find_by_av(avs->CODECPAR->codec_id); if (!src_pt.codec_def) { ilog(LOG_ERR, "Attempting to play media from an unsupported file format/codec"); return -1; } src_pt.encoding = src_pt.codec_def->rtpname_str; src_pt.channels = avs->CODECPAR->channels; src_pt.clock_rate = avs->CODECPAR->sample_rate; codec_init_payload_type(&src_pt, mp->media); // find suitable output payload type struct rtp_payload_type *dst_pt; for (GList *l = mp->media->codecs_prefs_send.head; l; l = l->next) { dst_pt = l->data; if (dst_pt->codec_def && !dst_pt->codec_def->supplemental) goto found; } dst_pt = NULL; found: if (!dst_pt) { ilog(LOG_ERR, "No supported output codec found in SDP"); return -1; } ilog(LOG_DEBUG, "Output codec for media playback is " STR_FORMAT, STR_FMT(&dst_pt->encoding_with_params)); // if we played anything before, scale our sync TS according to the time // that has passed if (mp->sync_ts_tv.tv_sec) { long long ts_diff_us = timeval_diff(&rtpe_now, &mp->sync_ts_tv); mp->sync_ts += ts_diff_us * dst_pt->clock_rate / 1000000 / dst_pt->codec_def->clockrate_mult; } mp->handler = codec_handler_make_playback(&src_pt, dst_pt, mp->sync_ts); if (!mp->handler) return -1; mp->duration = avs->duration * 1000 * avs->time_base.num / avs->time_base.den; return 0; } // appropriate lock must be held static void media_player_read_packet(struct media_player *mp) { if (!mp->fmtctx) return; int ret = av_read_frame(mp->fmtctx, &mp->pkt); if (ret < 0) { if (ret == AVERROR_EOF) { ilog(LOG_DEBUG, "EOF reading from media stream"); return; } ilog(LOG_ERR, "Error while reading from media stream"); return; } if (!mp->fmtctx->streams) { ilog(LOG_ERR, "No AVStream present in format context"); goto out; } AVStream *avs = mp->fmtctx->streams[0]; if (!avs) { ilog(LOG_ERR, "No AVStream present in format context"); goto out; } if (__ensure_codec_handler(mp, avs)) goto out; // scale pts and duration according to sample rate long long duration_scaled = mp->pkt.duration * avs->CODECPAR->sample_rate * avs->time_base.num / avs->time_base.den; unsigned long long pts_scaled = mp->pkt.pts * avs->CODECPAR->sample_rate * avs->time_base.num / avs->time_base.den; long long us_dur = mp->pkt.duration * 1000000LL * avs->time_base.num / avs->time_base.den; ilog(LOG_DEBUG, "read media packet: pts %llu duration %lli (scaled %llu/%lli, %lli us), " "sample rate %i, time_base %i/%i", (unsigned long long) mp->pkt.pts, (long long) mp->pkt.duration, pts_scaled, duration_scaled, us_dur, avs->CODECPAR->sample_rate, avs->time_base.num, avs->time_base.den); // synthesise fake RTP header and media_packet context struct rtp_header rtp = { .timestamp = pts_scaled, // taken verbatim by handler_func_playback w/o byte swap .seq_num = htons(mp->seq), }; struct media_packet packet = { .tv = rtpe_now, .call = mp->call, .media = mp->media, .rtp = &rtp, .ssrc_out = mp->ssrc_out, }; str_init_len(&packet.raw, (char *) mp->pkt.data, mp->pkt.size); packet.payload = packet.raw; mp->handler->func(mp->handler, &packet); // as this is timing sensitive and we may have spent some time decoding, // update our global "now" timestamp gettimeofday(&rtpe_now, NULL); // keep track of RTP timestamps and real clock. look at the last packet we received // and update our sync TS. if (packet.packets_out.head) { struct codec_packet *p = packet.packets_out.head->data; if (p->rtp) { mp->sync_ts = ntohl(p->rtp->timestamp); mp->sync_ts_tv = p->to_send; } } media_packet_encrypt(mp->crypt_handler->out->rtp_crypt, mp->sink, &packet); mutex_lock(&mp->sink->out_lock); if (media_socket_dequeue(&packet, mp->sink)) ilog(LOG_ERR, "Error sending playback media to RTP sink"); mutex_unlock(&mp->sink->out_lock); timeval_add_usec(&mp->next_run, us_dur); timerthread_obj_schedule_abs(&mp->tt_obj, &mp->next_run); out: av_packet_unref(&mp->pkt); } // call->master_lock held in W static int media_player_play_init(struct media_player *mp) { media_player_shutdown(mp); // find call media suitable for playback struct call_media *media; for (GList *l = mp->ml->medias.head; l; l = l->next) { media = l->data; if (media->type_id != MT_AUDIO) continue; if (!MEDIA_ISSET(media, SEND)) continue; if (media->streams.length == 0) continue; goto found; } media = NULL; found: if (!media) { ilog(LOG_ERR, "No suitable SDP section for media playback"); return -1; } mp->media = media; mp->sink = media->streams.head->data; mp->crypt_handler = determine_handler(&transport_protocols[PROTO_RTP_AVP], media, 1); return 0; } // call->master_lock held in W static void media_player_play_start(struct media_player *mp) { // needed to have usable duration for some formats. ignore errors. avformat_find_stream_info(mp->fmtctx, NULL); mp->next_run = rtpe_now; // give ourselves a bit of a head start with decoding timeval_add_usec(&mp->next_run, -50000); media_player_read_packet(mp); } #endif // call->master_lock held in W int media_player_play_file(struct media_player *mp, const str *file) { #ifdef WITH_TRANSCODING if (media_player_play_init(mp)) return -1; char file_s[PATH_MAX]; snprintf(file_s, sizeof(file_s), STR_FORMAT, STR_FMT(file)); int ret = avformat_open_input(&mp->fmtctx, file_s, NULL, NULL); if (ret < 0) { ilog(LOG_ERR, "Failed to open media file for playback: %s", av_error(ret)); return -1; } media_player_play_start(mp); return 0; #else return -1; #endif } #ifdef WITH_TRANSCODING static int __mp_avio_read_wrap(void *opaque, uint8_t *buf, int buf_size) { struct media_player *mp = opaque; if (buf_size < 0) return AVERROR(EINVAL); if (buf_size == 0) return 0; if (!mp->read_pos.len) return AVERROR_EOF; int len = buf_size; if (len > mp->read_pos.len) len = mp->read_pos.len; memcpy(buf, mp->read_pos.s, len); str_shift(&mp->read_pos, len); return len; } static int __mp_avio_read(void *opaque, uint8_t *buf, int buf_size) { ilog(LOG_DEBUG, "__mp_avio_read(%i)", buf_size); int ret = __mp_avio_read_wrap(opaque, buf, buf_size); ilog(LOG_DEBUG, "__mp_avio_read(%i) = %i", buf_size, ret); return ret; }
int capt_capt(opt_t *opt) { int frame_size; unsigned int frame_num; unsigned char *frame_buf_yuv, *frame_buf_rgb, *jpeg_buf; struct timeval start; const int fps = opt->fps_opt; const int wait_flag = (opt->fps_opt < opt->fps_dev); const int w = opt->width, h = opt->height; int q = 75; /* jpeg compress quality */ int len; char mh[PIPE_HEADER_LEN]; /* EMON system Message Header */ unsigned int ts; /* timestamp in Message Header */ frame_size = mchip_hsize() * mchip_vsize() * 3; frame_buf_yuv = (unsigned char*)malloc(frame_size); frame_buf_rgb = (unsigned char*)malloc(frame_size); jpeg_buf = (unsigned char*)malloc(frame_size); /* allocate enougth memory for jpeg_buf */ if (frame_buf_yuv == NULL || frame_buf_rgb == NULL ||jpeg_buf == NULL){ e_printf("cannot malloc for frame_buf or jpeg_buf\n"); return -1; } stat_init(&STAT); ts = rand() * opt->freq; mchip_continuous_start(); gettimeofday(&start, NULL); STAT.start = start; /* copy struct timeval */ d2_printf("\njpegcapt: %ld.%06ld: wait_flag=%d", start.tv_sec, start.tv_usec, wait_flag); for (frame_num = 0; opt->max_frame == 0 || frame_num < opt->max_frame; frame_num++, ts += opt->freq){ struct timeval c, b, a; /* capture, before(encoding), after */ int d1, d2; if (debug_level > 0 && (frame_num % opt->stat_freq)== 0){ stat_print(&STAT, frame_num); } if (wait_proper_time(&start, fps, frame_num, 1) < 0){ STAT.skip_count++; continue; /* skip capture because it's too late */ } STAT.capt_count++; gettimeofday(&c, NULL); mchip_continuous_read(frame_buf_yuv, mchip_hsize()*mchip_vsize()*2); yuv_convert(frame_buf_yuv, frame_buf_rgb, mchip_hsize(), mchip_vsize()); gettimeofday(&b, NULL); len = jpeg_encode(frame_buf_rgb, jpeg_buf, w, h, q); gettimeofday(&a, NULL); d1 = timeval_diff_usec(&b, &c); d2 = timeval_diff_usec(&a, &b); timeval_add_usec(&STAT.capt_total, d1); timeval_add_usec(&STAT.jpgenc_total, d2); d3_printf("\n frame=%d, ts=%d, jpg_len=%d, q=%d" ", t1=%d, t2=%d", frame_num, ts, len, q, d1, d2); if (len > opt->dsize ){ q *= 0.75; continue; /* skip this picture */ }else if (len < opt->dsize * 0.9 && q < 90) { q++; } bzero(&mh, PIPE_HEADER_LEN); pipe_set_version(&mh, 1); pipe_set_marker(&mh, 1); pipe_set_length(&mh, len); pipe_set_timestamp(&mh, ts); if (pipe_blocked_write_block(STDOUT, &mh, jpeg_buf) ==PIPE_ERROR){ d1_printf("\npipe_blocked_write_block error!!" "len=%d, ts=%ud", len, ts); } else { STAT.out_count++; } } if (debug_level > 0){ stat_print(&STAT, frame_num); } return 0; }
static void timeval_add_usec_check_(struct timeval *a, int64_t diff, struct timeval const *expected) { timeval_add_usec(a, diff); assert(0 == timeval_cmp(a, expected)); }