static void jbuf_clear(JitterBuffer *q) { for (; q->bottom != q->top; ++q->bottom) { if (q->queue[q->bottom % q->size]) { rtp_free_msg(NULL, q->queue[q->bottom % q->size]); q->queue[q->bottom % q->size] = NULL; } } }
/** * Parses data into RTPMessage struct. Stores headers separately from the payload data * and so the length variable is set accordingly. */ RTPMessage *msg_parse ( const uint8_t *data, int length ) { RTPMessage *retu = calloc(1, sizeof (RTPMessage)); retu->header = extract_header ( data, length ); /* It allocates memory and all */ if ( !retu->header ) { LOGGER_WARNING("Header failed to extract!"); free(retu); return NULL; } uint16_t from_pos = retu->header->length; retu->length = length - from_pos; if ( GET_FLAG_EXTENSION ( retu->header ) ) { retu->ext_header = extract_ext_header ( data + from_pos, length ); if ( retu->ext_header ) { retu->length -= ( 4 /* Minimum ext header len */ + retu->ext_header->length * size_32 ); from_pos += ( 4 /* Minimum ext header len */ + retu->ext_header->length * size_32 ); } else { /* Error */ LOGGER_WARNING("Ext Header failed to extract!"); rtp_free_msg(NULL, retu); return NULL; } } else { retu->ext_header = NULL; } if ( length - from_pos <= MAX_RTP_SIZE ) memcpy ( retu->data, data + from_pos, length - from_pos ); else { LOGGER_WARNING("Invalid length!"); rtp_free_msg(NULL, retu); return NULL; } retu->next = NULL; return retu; }
int rtp_send_msg ( RTPSession *session, Messenger *messenger, const uint8_t *data, uint16_t length ) { RTPMessage *msg = rtp_new_message (session, data, length); if ( !msg ) return -1; if ( -1 == send_custom_lossy_packet(messenger, session->dest, msg->data, msg->length) ) { LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", length, strerror(errno)); rtp_free_msg ( session, msg ); return rtp_ErrorSending; } /* Set sequ number */ session->sequnum = session->sequnum >= MAX_SEQU_NUM ? 0 : session->sequnum + 1; rtp_free_msg ( session, msg ); return 0; }
int empty_queue(struct jitter_buffer *q) { while (q->size > 0) { rtp_free_msg(NULL, q->queue[q->front]); q->front++; if (q->front == q->capacity) q->front = 0; q->size--; } q->id_set = 0; q->queue_ready = 0; return 0; }
/** * @brief Receive RTP payload. * * @param av Handler. * @param type Type of the payload. * @param dest Storage. * @return int * @retval ToxAvError On Error. * @retval >=0 Size of received payload. */ inline__ int toxav_recv_rtp_payload ( ToxAv *av, int32_t call_index, ToxAvCallType type, uint8_t *dest ) { if ( !dest ) return ErrorInternal; if (cii(call_index, av->msi_session)) return ErrorNoCall; CallSpecific *call = &av->calls[call_index]; if ( !call->crtps[type - TypeAudio] ) return ErrorNoRtpSession; RTPMessage *message; if ( type == TypeAudio ) { do { message = rtp_recv_msg(call->crtps[audio_index]); if (message) { /* push the packet into the queue */ queue(call->j_buf, message); } } while (message); int success = 0; message = dequeue(call->j_buf, &success); if ( success == 2) return ErrorAudioPacketLost; } else { message = rtp_recv_msg(call->crtps[video_index]); } if ( message ) { memcpy(dest, message->data, message->length); int length = message->length; rtp_free_msg(NULL, message); return length; } return 0; }
void ac_iterate(ACSession *ac) { if (!ac) { return; } /* TODO(mannol): fix this and jitter buffering */ /* Enough space for the maximum frame size (120 ms 48 KHz stereo audio) */ int16_t tmp[5760 * 2]; struct RTPMessage *msg; int rc = 0; pthread_mutex_lock(ac->queue_mutex); while ((msg = jbuf_read((struct JitterBuffer *)ac->j_buf, &rc)) || rc == 2) { pthread_mutex_unlock(ac->queue_mutex); if (rc == 2) { LOGGER_DEBUG(ac->log, "OPUS correction"); int fs = (ac->lp_sampling_rate * ac->lp_frame_duration) / 1000; rc = opus_decode(ac->decoder, NULL, 0, tmp, fs, 1); } else { /* Get values from packet and decode. */ /* NOTE: This didn't work very well */ #if 0 rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); if (rc != -1) { cs->last_packet_sampling_rate = rc; } else { LOGGER_WARNING(ac->log, "Failed to load packet values!"); rtp_free_msg(msg); continue; } #endif /* Pick up sampling rate from packet */ memcpy(&ac->lp_sampling_rate, msg->data, 4); ac->lp_sampling_rate = net_ntohl(ac->lp_sampling_rate); ac->lp_channel_count = opus_packet_get_nb_channels(msg->data + 4); /** NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa, * it didn't work quite well. */ if (!reconfigure_audio_decoder(ac, ac->lp_sampling_rate, ac->lp_channel_count)) { LOGGER_WARNING(ac->log, "Failed to reconfigure decoder!"); free(msg); continue; } rc = opus_decode(ac->decoder, msg->data + 4, msg->len - 4, tmp, 5760, 0); free(msg); } if (rc < 0) { LOGGER_WARNING(ac->log, "Decoding error: %s", opus_strerror(rc)); } else if (ac->acb.first) { ac->lp_frame_duration = (rc * 1000) / ac->lp_sampling_rate; ac->acb.first(ac->av, ac->friend_number, tmp, rc, ac->lp_channel_count, ac->lp_sampling_rate, ac->acb.second); } return; } pthread_mutex_unlock(ac->queue_mutex); }
/* Called from RTP */ void queue_message(RTPSession *session, RTPMessage *msg) { /* This function is unregistered during call termination befor destroing * Codec session so no need to check for validity of cs */ CSSession *cs = session->cs; if (!cs) return; /* Audio */ if (session->payload_type == msi_TypeAudio % 128) { pthread_mutex_lock(cs->queue_mutex); int ret = jbuf_write(cs->j_buf, msg); pthread_mutex_unlock(cs->queue_mutex); if (ret == -1) { rtp_free_msg(NULL, msg); } } /* Video */ else { uint8_t *packet = msg->data; uint32_t packet_size = msg->length; if (packet_size < VIDEOFRAME_HEADER_SIZE) goto end; uint8_t diff = packet[0] - cs->frameid_in; if (diff != 0) { if (diff < 225) { /* New frame */ /* Flush last frames' data and get ready for this frame */ Payload *p = malloc(sizeof(Payload) + cs->frame_size); if (p) { pthread_mutex_lock(cs->queue_mutex); if (buffer_full(cs->vbuf_raw)) { LOGGER_DEBUG("Dropped video frame"); Payload *tp; buffer_read(cs->vbuf_raw, &tp); free(tp); } else { p->size = cs->frame_size; memcpy(p->data, cs->frame_buf, cs->frame_size); } buffer_write(cs->vbuf_raw, p); pthread_mutex_unlock(cs->queue_mutex); } else { LOGGER_WARNING("Allocation failed! Program might misbehave!"); goto end; } cs->last_timestamp = msg->header->timestamp; cs->frameid_in = packet[0]; memset(cs->frame_buf, 0, cs->frame_size); cs->frame_size = 0; } else { /* Old frame; drop */ LOGGER_DEBUG("Old packet: %u", packet[0]); goto end; } } uint8_t piece_number = packet[1]; uint32_t length_before_piece = ((piece_number - 1) * cs->video_frame_piece_size); uint32_t framebuf_new_length = length_before_piece + (packet_size - VIDEOFRAME_HEADER_SIZE); if (framebuf_new_length > cs->max_video_frame_size) { goto end; } /* Otherwise it's part of the frame so just process */ /* LOGGER_DEBUG("Video Packet: %u %u", packet[0], packet[1]); */ memcpy(cs->frame_buf + length_before_piece, packet + VIDEOFRAME_HEADER_SIZE, packet_size - VIDEOFRAME_HEADER_SIZE); if (framebuf_new_length > cs->frame_size) { cs->frame_size = framebuf_new_length; } end: rtp_free_msg(NULL, msg); } }
void cs_do(CSSession *cs) { /* Codec session should always be protected by call mutex so no need to check for cs validity */ if (!cs) return; Payload *p; int rc; int success = 0; pthread_mutex_lock(cs->queue_mutex); RTPMessage *msg; while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { pthread_mutex_unlock(cs->queue_mutex); uint16_t fsize = ((cs->audio_decoder_sample_rate * cs->audio_decoder_frame_duration) / 1000); int16_t tmp[fsize * cs->audio_decoder_channels]; if (success == 2) { rc = opus_decode(cs->audio_decoder, 0, 0, tmp, fsize, 1); } else { rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, fsize, 0); rtp_free_msg(NULL, msg); } if (rc < 0) { LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); } else if (cs->acb.first) { /* Play */ cs->acb.first(cs->agent, cs->call_idx, tmp, rc, cs->acb.second); } pthread_mutex_lock(cs->queue_mutex); } if (cs->vbuf_raw && !buffer_empty(cs->vbuf_raw)) { /* Decode video */ buffer_read(cs->vbuf_raw, &p); /* Leave space for (possibly) other thread to queue more data after we read it here */ pthread_mutex_unlock(cs->queue_mutex); rc = vpx_codec_decode(&cs->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US); free(p); if (rc != VPX_CODEC_OK) { LOGGER_ERROR("Error decoding video: %s", vpx_codec_err_to_string(rc)); } else { vpx_codec_iter_t iter = NULL; vpx_image_t *dest = vpx_codec_get_frame(&cs->v_decoder, &iter); /* Play decoded images */ for (; dest; dest = vpx_codec_get_frame(&cs->v_decoder, &iter)) { if (cs->vcb.first) cs->vcb.first(cs->agent, cs->call_idx, dest, cs->vcb.second); vpx_img_free(dest); } } return; } pthread_mutex_unlock(cs->queue_mutex); }
void toxav_handle_packet(RTPSession *_session, RTPMessage *_msg) { ToxAv *av = _session->av; int32_t call_index = _session->call_index; CallSpecific *call = &av->calls[call_index]; if (!call->call_active) return; if (_session->payload_type == type_audio % 128) { queue(call->j_buf, _msg); int success = 0; while ((_msg = dequeue(call->j_buf, &success)) || success == 2) { DECODE_PACKET *p; if (success == 2) { p = malloc(sizeof(DECODE_PACKET)); if (p) { p->size = 0; } } else { p = malloc(sizeof(DECODE_PACKET) + _msg->length); if (p) { p->size = _msg->length; memcpy(p->data, _msg->data, _msg->length); } rtp_free_msg(NULL, _msg); } if (p) { /* do the decoding on another thread */ pthread_mutex_lock(&call->decode_cond_mutex); uint8_t w = call->audio_decode_write; if (call->audio_decode_queue[w] == NULL) { call->audio_decode_queue[w] = p; call->audio_decode_write = (w + 1) % AUDIO_DECODE_QUEUE_SIZE; pthread_cond_signal(&call->decode_cond); } else { LOGGER_DEBUG("Dropped audio frame\n"); free(p); } pthread_mutex_unlock(&call->decode_cond_mutex); } else { //malloc failed } } } else { uint8_t *packet = _msg->data; int recved_size = _msg->length; if (recved_size < VIDEOFRAME_HEADER_SIZE) { goto end; } uint8_t i = packet[0] - call->frame_id; if (i == 0) { /* piece of current frame */ } else if (i > 0 && i < 128) { /* received a piece of a frame ahead, flush current frame and start reading this new frame */ DECODE_PACKET *p = malloc(sizeof(DECODE_PACKET) + call->frame_limit); if (p) { p->size = call->frame_limit; memcpy(p->data, call->frame_buf, call->frame_limit); /* do the decoding on another thread */ pthread_mutex_lock(&call->decode_cond_mutex); uint8_t w = call->video_decode_write; if (call->video_decode_queue[w] == NULL) { call->video_decode_queue[w] = p; call->video_decode_write = (w + 1) % VIDEO_DECODE_QUEUE_SIZE; pthread_cond_signal(&call->decode_cond); } else { LOGGER_DEBUG("Dropped video frame\n"); free(p); } pthread_mutex_unlock(&call->decode_cond_mutex); } else { //malloc failed } call->frame_id = packet[0]; memset(call->frame_buf, 0, call->frame_limit); call->frame_limit = 0; } else { /* old packet, dont read */ LOGGER_DEBUG("Old packet: %u\n", i); goto end; } if (packet[1] > (MAX_VIDEOFRAME_SIZE - VIDEOFRAME_PIECE_SIZE + 1) / VIDEOFRAME_PIECE_SIZE) { //TODO, fix this check? not sure /* packet out of buffer range */ goto end; } LOGGER_DEBUG("Video Packet: %u %u\n", packet[0], packet[1]); memcpy(call->frame_buf + packet[1] * VIDEOFRAME_PIECE_SIZE, packet + VIDEOFRAME_HEADER_SIZE, recved_size - VIDEOFRAME_HEADER_SIZE); uint32_t limit = packet[1] * VIDEOFRAME_PIECE_SIZE + recved_size - VIDEOFRAME_HEADER_SIZE; if (limit > call->frame_limit) { call->frame_limit = limit; LOGGER_DEBUG("Limit: %u\n", call->frame_limit); } end: ; rtp_free_msg(NULL, _msg); } }
int main( int argc, char* argv[] ) { int status; tox_IP_Port Ip_port; const char* ip, *psend, *plisten; uint16_t port_send, port_listen; const uint8_t* test_bytes = "0123456789012345678901234567890123456789012345678901234567890123456789" "0123456789012345678901234567890123456789012345678901234567890123456789" "0123456789012345678901234567890123456789012345678901234567890123456789" "0123456789012345678901234567890123456789012345678901234567890123456789"; rtp_session_t* _m_session; rtp_msg_t *_m_msg_R, *_m_msg_S; arg_t* _list = parse_args ( argc, argv ); ip = find_arg_duble(_list, "-d"); psend = find_arg_duble(_list, "-p"); plisten = find_arg_duble(_list, "-l"); if ( !ip || !plisten || !psend ) return _print_help(argv[0]); port_send = atoi(psend); port_listen = atoi(plisten); IP_Port local, remote; /* * This is the Local ip. We initiate networking on * this value for it's the local one. To make stuff simpler we receive over this value * and send on the other one ( see remote ) */ local.ip.i = htonl(INADDR_ANY); local.port = port_listen; Networking_Core* _networking = new_networking(local.ip, port_listen); if ( !_networking ) return FAILURE; int _socket = _networking->sock; /* * Now this is the remote. It's used by rtp_session_t to determine the receivers ip etc. */ t_setipport ( ip, port_send, &remote ); _m_session = rtp_init_session(-1, -1); rtp_add_receiver( _m_session, &remote ); /* Now let's start our main loop in both recv and send mode */ for ( ;; ) { /* * This part checks for received messages and if gotten one * display 'Received msg!' indicator and free message */ _m_msg_R = rtp_recv_msg ( _m_session ); if ( _m_msg_R ) { puts ( "Received msg!" ); rtp_free_msg(_m_session, _m_msg_R); } /* -------------------- */ /* * This one makes a test msg and sends that message to the 'remote' */ _m_msg_S = rtp_msg_new ( _m_session, test_bytes, 280 ) ; rtp_send_msg ( _m_session, _m_msg_S, _socket ); usleep ( 10000 ); /* -------------------- */ } return SUCCESS; }