bool reconfigure_audio_decoder(ACSession *ac, int32_t sampling_rate, int8_t channels)
{
if (sampling_rate != ac->ld_sample_rate || channels != ac->ld_channel_count) {
if (current_time_monotonic() - ac->ldrts < 500) {
return false;
}
int status;
OpusDecoder *new_dec = opus_decoder_create(sampling_rate, channels, &status);
if (status != OPUS_OK) {
LOGGER_ERROR(ac->log, "Error while starting audio decoder(%d %d): %s", sampling_rate, channels, opus_strerror(status));
return false;
}
ac->ld_sample_rate = sampling_rate;
ac->ld_channel_count = channels;
ac->ldrts = current_time_monotonic();
opus_decoder_destroy(ac->decoder);
ac->decoder = new_dec;
LOGGER_DEBUG(ac->log, "Reconfigured audio decoder sr: %d cc: %d", sampling_rate, channels);
}
return true;
}
void toxav_iterate(ToxAV *av)
{
pthread_mutex_lock(av->mutex);
if (av->calls == NULL) {
pthread_mutex_unlock(av->mutex);
return;
}
uint64_t start = current_time_monotonic();
int32_t rc = 500;
ToxAVCall *i = av->calls[av->calls_head];
for (; i; i = i->next) {
if (i->active) {
pthread_mutex_lock(i->mutex);
pthread_mutex_unlock(av->mutex);
ac_iterate(i->audio.second);
vc_iterate(i->video.second);
if (i->msi_call->self_capabilities & msi_CapRAudio &&
i->msi_call->peer_capabilities & msi_CapSAudio)
rc = MIN(i->audio.second->lp_frame_duration, rc);
if (i->msi_call->self_capabilities & msi_CapRVideo &&
i->msi_call->peer_capabilities & msi_CapSVideo)
rc = MIN(i->video.second->lcfd, (uint32_t) rc);
uint32_t fid = i->friend_number;
pthread_mutex_unlock(i->mutex);
pthread_mutex_lock(av->mutex);
/* In case this call is popped from container stop iteration */
if (call_get(av, fid) != i)
break;
}
}
pthread_mutex_unlock(av->mutex);
av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa);
av->dmsst += current_time_monotonic() - start;
if (++av->dmssc == 3) {
av->dmssa = av->dmsst / 3 + 5 /* NOTE Magic Offset for precission */;
av->dmssc = 0;
av->dmsst = 0;
}
}
int reset_timer_event ( int id, uint32_t timeout )
{
int _status;
LOCK((&event_handler));
EventContainer *_it = event_handler.timed_events;
int _i = event_handler.timed_events_count;
/* Find it and change */
for ( ; _i; _i-- ) {
if ( _it->id == id ) {
_it->timeout = timeout + ((uint32_t)current_time_monotonic());
break;
}
++_it;
}
_status = _i ? -1 : 0;
UNLOCK((&event_handler));
return _status;
}
/**
* Builds header from control session values.
*/
RTPHeader *build_header ( RTPSession *session )
{
RTPHeader *retu = calloc ( 1, sizeof (RTPHeader) );
if ( !retu ) {
LOGGER_WARNING("Alloc failed! Program might misbehave!");
return NULL;
}
ADD_FLAG_VERSION ( retu, session->version );
ADD_FLAG_PADDING ( retu, session->padding );
ADD_FLAG_EXTENSION ( retu, session->extension );
ADD_FLAG_CSRCC ( retu, session->cc );
ADD_SETTING_MARKER ( retu, session->marker );
ADD_SETTING_PAYLOAD ( retu, session->payload_type );
retu->sequnum = session->sequnum;
retu->timestamp = current_time_monotonic(); /* milliseconds */
retu->ssrc = session->ssrc;
int i;
for ( i = 0; i < session->cc; i++ )
retu->csrc[i] = session->csrc[i];
retu->length = 12 /* Minimum header len */ + ( session->cc * size_32 );
return retu;
}
static int handle_fakeid_announce(void *object, const uint8_t *source_pubkey, const uint8_t *data, uint32_t length)
{
Onion_Client *onion_c = object;
if (length < FAKEID_DATA_MIN_LENGTH)
return 1;
if (length > FAKEID_DATA_MAX_LENGTH)
return 1;
int friend_num = onion_friend_num(onion_c, source_pubkey);
if (friend_num == -1)
return 1;
uint64_t no_replay;
memcpy(&no_replay, data + 1, sizeof(uint64_t));
net_to_host((uint8_t *) &no_replay, sizeof(no_replay));
if (no_replay <= onion_c->friends_list[friend_num].last_noreplay)
return 1;
onion_c->friends_list[friend_num].last_noreplay = no_replay;
onion_set_friend_DHT_pubkey(onion_c, friend_num, data + 1 + sizeof(uint64_t), current_time_monotonic());
onion_c->friends_list[friend_num].last_seen = unix_time();
uint16_t len_nodes = length - FAKEID_DATA_MIN_LENGTH;
if (len_nodes != 0) {
Node_format nodes[MAX_SENT_NODES];
int num_nodes = unpack_nodes(nodes, MAX_SENT_NODES, 0, data + 1 + sizeof(uint64_t) + crypto_box_PUBLICKEYBYTES,
len_nodes, 1);
if (num_nodes <= 0)
return 1;
int i;
for (i = 0; i < num_nodes; ++i) {
uint8_t family = nodes[i].ip_port.ip.family;
if (family == AF_INET || family == AF_INET6) {
DHT_getnodes(onion_c->dht, &nodes[i].ip_port, nodes[i].client_id, onion_c->friends_list[friend_num].fake_client_id);
} else if (family == TCP_INET || family == TCP_INET6) {
if (onion_c->friends_list[friend_num].tcp_relay_node_callback) {
void *obj = onion_c->friends_list[friend_num].tcp_relay_node_callback_object;
uint32_t number = onion_c->friends_list[friend_num].tcp_relay_node_callback_number;
onion_c->friends_list[friend_num].tcp_relay_node_callback(obj, number, nodes[i].ip_port, nodes[i].client_id);
}
}
}
}
return 0;
}
int vc_queue_message(Mono_Time *mono_time, void *vcp, struct RTPMessage *msg)
{
/* This function is called with complete messages
* they have already been assembled.
* this function gets called from handle_rtp_packet() and handle_rtp_packet_v3()
*/
if (!vcp || !msg) {
return -1;
}
VCSession *vc = (VCSession *)vcp;
const struct RTPHeader *const header = &msg->header;
if (msg->header.pt == (RTP_TYPE_VIDEO + 2) % 128) {
LOGGER_WARNING(vc->log, "Got dummy!");
free(msg);
return 0;
}
if (msg->header.pt != RTP_TYPE_VIDEO % 128) {
LOGGER_WARNING(vc->log, "Invalid payload type! pt=%d", (int)msg->header.pt);
free(msg);
return -1;
}
pthread_mutex_lock(vc->queue_mutex);
if ((header->flags & RTP_LARGE_FRAME) && header->pt == RTP_TYPE_VIDEO % 128) {
LOGGER_DEBUG(vc->log, "rb_write msg->len=%d b0=%d b1=%d", (int)msg->len, (int)msg->data[0], (int)msg->data[1]);
}
free(rb_write(vc->vbuf_raw, msg));
/* Calculate time it took for peer to send us this frame */
uint32_t t_lcfd = current_time_monotonic(mono_time) - vc->linfts;
vc->lcfd = t_lcfd > 100 ? vc->lcfd : t_lcfd;
vc->linfts = current_time_monotonic(mono_time);
pthread_mutex_unlock(vc->queue_mutex);
return 0;
}
/* Place and order array of timers */
int throw_timer_event ( void *(func)(void *), void *arg, unsigned timeout)
{
static int _unique_id = 1;
push_event(&event_handler.timed_events, &(event_handler.timed_events_count), func, arg );
size_t _counter = event_handler.timed_events_count;
event_handler.timed_events[_counter - 1].timeout = timeout + ((uint32_t)current_time_monotonic());
event_handler.timed_events[_counter - 1].id = _unique_id;
++_unique_id;
/* reorder */
reorder_events(_counter, event_handler.timed_events, timeout );
return _unique_id - 1;
}
/* main poll for event execution */
void *event_poll( void *arg )
{
EventHandler *_event_handler = arg;
while ( _event_handler->running ) {
LOCK( _event_handler );
if ( _event_handler->timed_events ) {
uint32_t _time = ((uint32_t)current_time_monotonic());
if ( _event_handler->timed_events[0].timeout < _time ) {
RUN_IN_THREAD ( _event_handler->timed_events[0].func,
_event_handler->timed_events[0].func_args );
pop_id(&_event_handler->timed_events,
&_event_handler->timed_events_count,
_event_handler->timed_events[0].id);
}
}
UNLOCK( _event_handler );
usleep(FREQUENCY);
}
LOCK( _event_handler );
clear_events(&_event_handler->timed_events, &_event_handler->timed_events_count);
UNLOCK( _event_handler );
_event_handler->running = -1;
pthread_exit(NULL);
}
/* Basic network functions:
* Function to send packet(data) of length length to ip_port.
*/
int sendpacket(Networking_Core *net, IP_Port ip_port, uint8_t *data, uint32_t length)
{
/* socket AF_INET, but target IP NOT: can't send */
if ((net->family == AF_INET) && (ip_port.ip.family != AF_INET))
return -1;
struct sockaddr_storage addr;
size_t addrsize = 0;
if (ip_port.ip.family == AF_INET) {
if (net->family == AF_INET6) {
/* must convert to IPV4-in-IPV6 address */
struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *)&addr;
addrsize = sizeof(struct sockaddr_in6);
addr6->sin6_family = AF_INET6;
addr6->sin6_port = ip_port.port;
/* there should be a macro for this in a standards compliant
* environment, not found */
IP6 ip6;
ip6.uint32[0] = 0;
ip6.uint32[1] = 0;
ip6.uint32[2] = htonl(0xFFFF);
ip6.uint32[3] = ip_port.ip.ip4.uint32;
addr6->sin6_addr = ip6.in6_addr;
addr6->sin6_flowinfo = 0;
addr6->sin6_scope_id = 0;
} else {
struct sockaddr_in *addr4 = (struct sockaddr_in *)&addr;
addrsize = sizeof(struct sockaddr_in);
addr4->sin_family = AF_INET;
addr4->sin_addr = ip_port.ip.ip4.in_addr;
addr4->sin_port = ip_port.port;
}
} else if (ip_port.ip.family == AF_INET6) {
struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *)&addr;
addrsize = sizeof(struct sockaddr_in6);
addr6->sin6_family = AF_INET6;
addr6->sin6_port = ip_port.port;
addr6->sin6_addr = ip_port.ip.ip6.in6_addr;
addr6->sin6_flowinfo = 0;
addr6->sin6_scope_id = 0;
} else {
/* unknown address type*/
return -1;
}
int res = sendto(net->sock, (char *) data, length, 0, (struct sockaddr *)&addr, addrsize);
loglogdata("O=>", data, length, ip_port, res);
if ((res >= 0) && ((uint32_t)res == length))
net->send_fail_eagain = 0;
else if ((res < 0) && (errno == EWOULDBLOCK))
net->send_fail_eagain = current_time_monotonic();
return res;
}
VCSession *vc_new(Mono_Time *mono_time, const Logger *log, ToxAV *av, uint32_t friend_number,
toxav_video_receive_frame_cb *cb, void *cb_data)
{
VCSession *vc = (VCSession *)calloc(sizeof(VCSession), 1);
vpx_codec_err_t rc;
if (!vc) {
LOGGER_WARNING(log, "Allocation failed! Application might misbehave!");
return nullptr;
}
if (create_recursive_mutex(vc->queue_mutex) != 0) {
LOGGER_WARNING(log, "Failed to create recursive mutex!");
free(vc);
return nullptr;
}
int cpu_used_value = VP8E_SET_CPUUSED_VALUE;
vc->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE);
if (!vc->vbuf_raw) {
goto BASE_CLEANUP;
}
/*
* VPX_CODEC_USE_FRAME_THREADING
* Enable frame-based multi-threading
*
* VPX_CODEC_USE_ERROR_CONCEALMENT
* Conceal errors in decoded frames
*/
vpx_codec_dec_cfg_t dec_cfg;
dec_cfg.threads = VPX_MAX_DECODER_THREADS; // Maximum number of threads to use
dec_cfg.w = VIDEO_CODEC_DECODER_MAX_WIDTH;
dec_cfg.h = VIDEO_CODEC_DECODER_MAX_HEIGHT;
LOGGER_DEBUG(log, "Using VP8 codec for decoder (0)");
rc = vpx_codec_dec_init(vc->decoder, video_codec_decoder_interface(), &dec_cfg,
VPX_CODEC_USE_FRAME_THREADING | VPX_CODEC_USE_POSTPROC);
if (rc == VPX_CODEC_INCAPABLE) {
LOGGER_WARNING(log, "Postproc not supported by this decoder (0)");
rc = vpx_codec_dec_init(vc->decoder, video_codec_decoder_interface(), &dec_cfg, VPX_CODEC_USE_FRAME_THREADING);
}
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR(log, "Init video_decoder failed: %s", vpx_codec_err_to_string(rc));
goto BASE_CLEANUP;
}
if (VIDEO_VP8_DECODER_POST_PROCESSING_ENABLED == 1) {
vp8_postproc_cfg_t pp = {VP8_DEBLOCK, 1, 0};
vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp);
if (cc_res != VPX_CODEC_OK) {
LOGGER_WARNING(log, "Failed to turn on postproc");
} else {
LOGGER_DEBUG(log, "turn on postproc: OK");
}
} else {
vp8_postproc_cfg_t pp = {0, 0, 0};
vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp);
if (cc_res != VPX_CODEC_OK) {
LOGGER_WARNING(log, "Failed to turn OFF postproc");
} else {
LOGGER_DEBUG(log, "Disable postproc: OK");
}
}
/* Set encoder to some initial values
*/
vpx_codec_enc_cfg_t cfg;
vc_init_encoder_cfg(log, &cfg, 1);
LOGGER_DEBUG(log, "Using VP8 codec for encoder (0.1)");
rc = vpx_codec_enc_init(vc->encoder, video_codec_encoder_interface(), &cfg, VPX_CODEC_USE_FRAME_THREADING);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR(log, "Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
goto BASE_CLEANUP_1;
}
rc = vpx_codec_control(vc->encoder, VP8E_SET_CPUUSED, cpu_used_value);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR(log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
vpx_codec_destroy(vc->encoder);
goto BASE_CLEANUP_1;
}
/*
VPX_CTRL_USE_TYPE(VP8E_SET_NOISE_SENSITIVITY, unsigned int)
control function to set noise sensitivity
0: off, 1: OnYOnly, 2: OnYUV, 3: OnYUVAggressive, 4: Adaptive
*/
/*
rc = vpx_codec_control(vc->encoder, VP8E_SET_NOISE_SENSITIVITY, 2);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR(log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
vpx_codec_destroy(vc->encoder);
goto BASE_CLEANUP_1;
}
*/
vc->linfts = current_time_monotonic(mono_time);
vc->lcfd = 60;
vc->vcb = cb;
vc->vcb_user_data = cb_data;
vc->friend_number = friend_number;
vc->av = av;
vc->log = log;
return vc;
BASE_CLEANUP_1:
vpx_codec_destroy(vc->decoder);
BASE_CLEANUP:
pthread_mutex_destroy(vc->queue_mutex);
rb_kill(vc->vbuf_raw);
free(vc);
return nullptr;
}
bool toxav_audio_send_frame(ToxAV *av, uint32_t friend_number, const int16_t *pcm, size_t sample_count,
uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME *error)
{
uint64_t start_time = current_time_monotonic();
uint64_t delta = start_time;
LOGGER_DEBUG ("Starting time: %llu", start_time);
TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK;
ToxAVCall *call;
if (m_friend_exists(av->m, friend_number) == 0) {
rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND;
goto END;
}
if (pthread_mutex_trylock(av->mutex) != 0) {
rc = TOXAV_ERR_SEND_FRAME_SYNC;
goto END;
}
call = call_get(av, friend_number);
if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) {
pthread_mutex_unlock(av->mutex);
rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL;
goto END;
}
if (call->audio_bit_rate == 0 ||
!(call->msi_call->self_capabilities & msi_CapSAudio) ||
!(call->msi_call->peer_capabilities & msi_CapRAudio)) {
pthread_mutex_unlock(av->mutex);
rc = TOXAV_ERR_SEND_FRAME_PAYLOAD_TYPE_DISABLED;
goto END;
}
pthread_mutex_lock(call->mutex_audio);
pthread_mutex_unlock(av->mutex);
if (pcm == NULL) {
pthread_mutex_unlock(call->mutex_audio);
rc = TOXAV_ERR_SEND_FRAME_NULL;
goto END;
}
if (channels > 2) {
pthread_mutex_unlock(call->mutex_audio);
rc = TOXAV_ERR_SEND_FRAME_INVALID;
goto END;
}
LOGGER_DEBUG("Initial checking took: %llu", current_time_monotonic() - delta);
delta = current_time_monotonic();
{ /* Encode and send */
if (ac_reconfigure_encoder(call->audio.second, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) {
pthread_mutex_unlock(call->mutex_audio);
rc = TOXAV_ERR_SEND_FRAME_INVALID;
goto END;
}
LOGGER_DEBUG("Encoder configuration took: %llu", current_time_monotonic() - delta);
delta = current_time_monotonic();
uint8_t dest[sample_count + sizeof(sampling_rate)]; /* This is more than enough always */
sampling_rate = htonl(sampling_rate);
memcpy(dest, &sampling_rate, sizeof(sampling_rate));
int vrc = opus_encode(call->audio.second->encoder, pcm, sample_count,
dest + sizeof(sampling_rate), sizeof(dest) - sizeof(sampling_rate));
LOGGER_DEBUG("Encoding took: %llu", current_time_monotonic() - delta);
delta = current_time_monotonic();
if (vrc < 0) {
LOGGER_WARNING("Failed to encode frame %s", opus_strerror(vrc));
pthread_mutex_unlock(call->mutex_audio);
rc = TOXAV_ERR_SEND_FRAME_INVALID;
goto END;
}
if (rtp_send_data(call->audio.first, dest, vrc + sizeof(sampling_rate)) != 0) {
LOGGER_WARNING("Failed to send audio packet");
rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED;
}
LOGGER_DEBUG("Sending took: %llu", current_time_monotonic() - delta);
}
pthread_mutex_unlock(call->mutex_audio);
END:
LOGGER_DEBUG ("End time: %llu Total: %llu", current_time_monotonic(), current_time_monotonic() - start_time);
if (error)
*error = rc;
return rc == TOXAV_ERR_SEND_FRAME_OK;
}