static GstFlowReturn opus_dec_chain_parse_data (GstOpusDec * dec, GstBuffer * buffer) { GstFlowReturn res = GST_FLOW_OK; gsize size; guint8 *data; GstBuffer *outbuf, *bufd; gint16 *out_data; int n, err; int samples; unsigned int packet_size; GstBuffer *buf; GstMapInfo map, omap; GstAudioClippingMeta *cmeta = NULL; if (dec->state == NULL) { /* If we did not get any headers, default to 2 channels */ if (dec->n_channels == 0) { GST_INFO_OBJECT (dec, "No header, assuming single stream"); dec->n_channels = 2; dec->sample_rate = 48000; /* default stereo mapping */ dec->channel_mapping_family = 0; dec->channel_mapping[0] = 0; dec->channel_mapping[1] = 1; dec->n_streams = 1; dec->n_stereo_streams = 1; if (!gst_opus_dec_negotiate (dec, NULL)) return GST_FLOW_NOT_NEGOTIATED; } if (dec->n_channels == 2 && dec->n_streams == 1 && dec->n_stereo_streams == 0) { /* if we are automatically decoding 2 channels, but only have a single encoded one, direct both channels to it */ dec->channel_mapping[1] = 0; } GST_DEBUG_OBJECT (dec, "Creating decoder with %d channels, %d Hz", dec->n_channels, dec->sample_rate); #ifndef GST_DISABLE_GST_DEBUG gst_opus_common_log_channel_mapping_table (GST_ELEMENT (dec), opusdec_debug, "Mapping table", dec->n_channels, dec->channel_mapping); #endif GST_DEBUG_OBJECT (dec, "%d streams, %d stereo", dec->n_streams, dec->n_stereo_streams); dec->state = opus_multistream_decoder_create (dec->sample_rate, dec->n_channels, dec->n_streams, dec->n_stereo_streams, dec->channel_mapping, &err); if (!dec->state || err != OPUS_OK) goto creation_failed; } if (buffer) { GST_DEBUG_OBJECT (dec, "Received buffer of size %" G_GSIZE_FORMAT, gst_buffer_get_size (buffer)); } else { GST_DEBUG_OBJECT (dec, "Received missing buffer"); } /* if using in-band FEC, we introdude one extra frame's delay as we need to potentially wait for next buffer to decode a missing buffer */ if (dec->use_inband_fec && !dec->primed) { GST_DEBUG_OBJECT (dec, "First buffer received in FEC mode, early out"); gst_buffer_replace (&dec->last_buffer, buffer); dec->primed = TRUE; goto done; } /* That's the buffer we'll be sending to the opus decoder. */ buf = (dec->use_inband_fec && gst_buffer_get_size (dec->last_buffer) > 0) ? dec->last_buffer : buffer; /* That's the buffer we get duration from */ bufd = dec->use_inband_fec ? dec->last_buffer : buffer; if (buf && gst_buffer_get_size (buf) > 0) { gst_buffer_map (buf, &map, GST_MAP_READ); data = map.data; size = map.size; GST_DEBUG_OBJECT (dec, "Using buffer of size %" G_GSIZE_FORMAT, size); } else { /* concealment data, pass NULL as the bits parameters */ GST_DEBUG_OBJECT (dec, "Using NULL buffer"); data = NULL; size = 0; } if (gst_buffer_get_size (bufd) == 0) { GstClockTime const opus_plc_alignment = 2500 * GST_USECOND; GstClockTime aligned_missing_duration; GstClockTime missing_duration = GST_BUFFER_DURATION (bufd); if (!GST_CLOCK_TIME_IS_VALID (missing_duration) || missing_duration == 0) { if (GST_CLOCK_TIME_IS_VALID (dec->last_known_buffer_duration)) { missing_duration = dec->last_known_buffer_duration; GST_WARNING_OBJECT (dec, "Missing duration, using last duration %" GST_TIME_FORMAT, GST_TIME_ARGS (missing_duration)); } else { GST_WARNING_OBJECT (dec, "Missing buffer, but unknown duration, and no previously known duration, assuming 20 ms"); missing_duration = 20 * GST_MSECOND; } } GST_DEBUG_OBJECT (dec, "missing buffer, doing PLC duration %" GST_TIME_FORMAT " plus leftover %" GST_TIME_FORMAT, GST_TIME_ARGS (missing_duration), GST_TIME_ARGS (dec->leftover_plc_duration)); /* add the leftover PLC duration to that of the buffer */ missing_duration += dec->leftover_plc_duration; /* align the combined buffer and leftover PLC duration to multiples * of 2.5ms, rounding to nearest, and store excess duration for later */ aligned_missing_duration = ((missing_duration + opus_plc_alignment / 2) / opus_plc_alignment) * opus_plc_alignment; dec->leftover_plc_duration = missing_duration - aligned_missing_duration; /* Opus' PLC cannot operate with less than 2.5ms; skip PLC * and accumulate the missing duration in the leftover_plc_duration * for the next PLC attempt */ if (aligned_missing_duration < opus_plc_alignment) { GST_DEBUG_OBJECT (dec, "current duration %" GST_TIME_FORMAT " of missing data not enough for PLC (minimum needed: %" GST_TIME_FORMAT ") - skipping", GST_TIME_ARGS (missing_duration), GST_TIME_ARGS (opus_plc_alignment)); goto done; } /* convert the duration (in nanoseconds) to sample count */ samples = gst_util_uint64_scale_int (aligned_missing_duration, dec->sample_rate, GST_SECOND); GST_DEBUG_OBJECT (dec, "calculated PLC frame length: %" GST_TIME_FORMAT " num frame samples: %d new leftover: %" GST_TIME_FORMAT, GST_TIME_ARGS (aligned_missing_duration), samples, GST_TIME_ARGS (dec->leftover_plc_duration)); } else { /* use maximum size (120 ms) as the number of returned samples is not constant over the stream. */ samples = 120 * dec->sample_rate / 1000; } packet_size = samples * dec->n_channels * 2; outbuf = gst_audio_decoder_allocate_output_buffer (GST_AUDIO_DECODER (dec), packet_size); if (!outbuf) { goto buffer_failed; } if (size > 0) dec->last_known_buffer_duration = packet_duration_opus (data, size); gst_buffer_map (outbuf, &omap, GST_MAP_WRITE); out_data = (gint16 *) omap.data; do { if (dec->use_inband_fec) { if (gst_buffer_get_size (dec->last_buffer) > 0) { /* normal delayed decode */ GST_LOG_OBJECT (dec, "FEC enabled, decoding last delayed buffer"); n = opus_multistream_decode (dec->state, data, size, out_data, samples, 0); } else { /* FEC reconstruction decode */ GST_LOG_OBJECT (dec, "FEC enabled, reconstructing last buffer"); n = opus_multistream_decode (dec->state, data, size, out_data, samples, 1); } } else { /* normal decode */ GST_LOG_OBJECT (dec, "FEC disabled, decoding buffer"); n = opus_multistream_decode (dec->state, data, size, out_data, samples, 0); } if (n == OPUS_BUFFER_TOO_SMALL) { /* if too small, add 2.5 milliseconds and try again, up to the * Opus max size of 120 milliseconds */ if (samples >= 120 * dec->sample_rate / 1000) break; samples += 25 * dec->sample_rate / 10000; packet_size = samples * dec->n_channels * 2; gst_buffer_unmap (outbuf, &omap); gst_buffer_unref (outbuf); outbuf = gst_audio_decoder_allocate_output_buffer (GST_AUDIO_DECODER (dec), packet_size); if (!outbuf) { goto buffer_failed; } gst_buffer_map (outbuf, &omap, GST_MAP_WRITE); out_data = (gint16 *) omap.data; } } while (n == OPUS_BUFFER_TOO_SMALL); gst_buffer_unmap (outbuf, &omap); if (data != NULL) gst_buffer_unmap (buf, &map); if (n < 0) { GstFlowReturn ret = GST_FLOW_ERROR; gst_buffer_unref (outbuf); GST_AUDIO_DECODER_ERROR (dec, 1, STREAM, DECODE, (NULL), ("Decoding error (%d): %s", n, opus_strerror (n)), ret); return ret; } GST_DEBUG_OBJECT (dec, "decoded %d samples", n); gst_buffer_set_size (outbuf, n * 2 * dec->n_channels); GST_BUFFER_DURATION (outbuf) = samples * GST_SECOND / dec->sample_rate; samples = n; cmeta = gst_buffer_get_audio_clipping_meta (buf); g_assert (!cmeta || cmeta->format == GST_FORMAT_DEFAULT); /* Skip any samples that need skipping */ if (cmeta && cmeta->start) { guint pre_skip = cmeta->start; guint scaled_pre_skip = pre_skip * dec->sample_rate / 48000; guint skip = scaled_pre_skip > n ? n : scaled_pre_skip; guint scaled_skip = skip * 48000 / dec->sample_rate; gst_buffer_resize (outbuf, skip * 2 * dec->n_channels, -1); GST_INFO_OBJECT (dec, "Skipping %u samples at the beginning (%u at 48000 Hz)", skip, scaled_skip); } if (cmeta && cmeta->end) { guint post_skip = cmeta->end; guint scaled_post_skip = post_skip * dec->sample_rate / 48000; guint skip = scaled_post_skip > n ? n : scaled_post_skip; guint scaled_skip = skip * 48000 / dec->sample_rate; guint outsize = gst_buffer_get_size (outbuf); guint skip_bytes = skip * 2 * dec->n_channels; if (outsize > skip_bytes) outsize -= skip_bytes; else outsize = 0; gst_buffer_resize (outbuf, 0, outsize); GST_INFO_OBJECT (dec, "Skipping %u samples at the end (%u at 48000 Hz)", skip, scaled_skip); } if (gst_buffer_get_size (outbuf) == 0) { gst_buffer_unref (outbuf); outbuf = NULL; } else if (dec->opus_pos[0] != GST_AUDIO_CHANNEL_POSITION_INVALID) { gst_audio_buffer_reorder_channels (outbuf, GST_AUDIO_FORMAT_S16, dec->n_channels, dec->opus_pos, dec->info.position); } /* Apply gain */ /* Would be better off leaving this to a volume element, as this is a naive conversion that does too many int/float conversions. However, we don't have control over the pipeline... So make it optional if the user program wants to use a volume, but do it by default so the correct volume goes out by default */ if (dec->apply_gain && outbuf && dec->r128_gain) { gsize rsize; unsigned int i, nsamples; double volume = dec->r128_gain_volume; gint16 *samples; gst_buffer_map (outbuf, &omap, GST_MAP_READWRITE); samples = (gint16 *) omap.data; rsize = omap.size; GST_DEBUG_OBJECT (dec, "Applying gain: volume %f", volume); nsamples = rsize / 2; for (i = 0; i < nsamples; ++i) { int sample = (int) (samples[i] * volume + 0.5); samples[i] = sample < -32768 ? -32768 : sample > 32767 ? 32767 : sample; } gst_buffer_unmap (outbuf, &omap); } if (dec->use_inband_fec) { gst_buffer_replace (&dec->last_buffer, buffer); } res = gst_audio_decoder_finish_frame (GST_AUDIO_DECODER (dec), outbuf, 1); if (res != GST_FLOW_OK) GST_DEBUG_OBJECT (dec, "flow: %s", gst_flow_get_name (res)); done: return res; creation_failed: GST_ELEMENT_ERROR (dec, LIBRARY, INIT, ("Failed to create Opus decoder"), ("Failed to create Opus decoder (%d): %s", err, opus_strerror (err))); return GST_FLOW_ERROR; buffer_failed: GST_ELEMENT_ERROR (dec, STREAM, DECODE, (NULL), ("Failed to create %u byte buffer", packet_size)); return GST_FLOW_ERROR; }
static GstFlowReturn gst_opus_parse_parse_frame (GstBaseParse * base, GstBaseParseFrame * frame) { guint64 duration; GstOpusParse *parse; gboolean is_idheader, is_commentheader; GstMapInfo map; GstAudioClippingMeta *cmeta = gst_buffer_get_audio_clipping_meta (frame->buffer); parse = GST_OPUS_PARSE (base); g_assert (!cmeta || cmeta->format == GST_FORMAT_DEFAULT); is_idheader = gst_opus_header_is_id_header (frame->buffer); is_commentheader = gst_opus_header_is_comment_header (frame->buffer); if (!parse->got_headers || !parse->header_sent) { GstCaps *caps; /* Opus streams can decode to 1 or 2 channels, so use the header value if we have one, or 2 otherwise */ if (is_idheader) { gst_buffer_replace (&parse->id_header, frame->buffer); GST_DEBUG_OBJECT (parse, "Found ID header, keeping"); return GST_BASE_PARSE_FLOW_DROPPED; } else if (is_commentheader) { gst_buffer_replace (&parse->comment_header, frame->buffer); GST_DEBUG_OBJECT (parse, "Found comment header, keeping"); return GST_BASE_PARSE_FLOW_DROPPED; } parse->got_headers = TRUE; if (cmeta && cmeta->start) { parse->pre_skip += cmeta->start; gst_buffer_map (frame->buffer, &map, GST_MAP_READ); duration = packet_duration_opus (map.data, map.size); gst_buffer_unmap (frame->buffer, &map); /* Queue frame for later once we know all initial padding */ if (duration == cmeta->start) { frame->flags |= GST_BASE_PARSE_FRAME_FLAG_QUEUE; } } if (!(frame->flags & GST_BASE_PARSE_FRAME_FLAG_QUEUE)) { if (FALSE && parse->id_header && parse->comment_header) { guint16 pre_skip; gst_buffer_map (parse->id_header, &map, GST_MAP_READWRITE); pre_skip = GST_READ_UINT16_LE (map.data + 10); if (pre_skip != parse->pre_skip) { GST_DEBUG_OBJECT (parse, "Fixing up pre-skip %u -> %" G_GUINT64_FORMAT, pre_skip, parse->pre_skip); GST_WRITE_UINT16_LE (map.data + 10, parse->pre_skip); } gst_buffer_unmap (parse->id_header, &map); caps = gst_codec_utils_opus_create_caps_from_header (parse->id_header, parse->comment_header); } else { GstCaps *sink_caps; guint32 sample_rate = 48000; guint8 n_channels, n_streams, n_stereo_streams, channel_mapping_family; guint8 channel_mapping[256]; GstBuffer *id_header; sink_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SINK_PAD (parse)); if (!sink_caps || !gst_codec_utils_opus_parse_caps (sink_caps, &sample_rate, &n_channels, &channel_mapping_family, &n_streams, &n_stereo_streams, channel_mapping)) { GST_INFO_OBJECT (parse, "No headers and no caps, blindly setting up canonical stereo"); n_channels = 2; n_streams = 1; n_stereo_streams = 1; channel_mapping_family = 0; channel_mapping[0] = 0; channel_mapping[1] = 1; } if (sink_caps) gst_caps_unref (sink_caps); id_header = gst_codec_utils_opus_create_header (sample_rate, n_channels, channel_mapping_family, n_streams, n_stereo_streams, channel_mapping, parse->pre_skip, 0); caps = gst_codec_utils_opus_create_caps_from_header (id_header, NULL); gst_buffer_unref (id_header); } gst_buffer_replace (&parse->id_header, NULL); gst_buffer_replace (&parse->comment_header, NULL); gst_pad_set_caps (GST_BASE_PARSE_SRC_PAD (parse), caps); gst_caps_unref (caps); parse->header_sent = TRUE; } } GST_BUFFER_TIMESTAMP (frame->buffer) = parse->next_ts; gst_buffer_map (frame->buffer, &map, GST_MAP_READ); duration = packet_duration_opus (map.data, map.size); gst_buffer_unmap (frame->buffer, &map); parse->next_ts += duration; GST_BUFFER_DURATION (frame->buffer) = duration; GST_BUFFER_OFFSET_END (frame->buffer) = gst_util_uint64_scale (parse->next_ts, 48000, GST_SECOND); GST_BUFFER_OFFSET (frame->buffer) = parse->next_ts; return GST_FLOW_OK; }