static int cook_decode_init(AVCodecContext *avctx) { COOKContext *q = avctx->priv_data; const uint8_t *edata_ptr = avctx->extradata; /* Take care of the codec specific extradata. */ if (avctx->extradata_size <= 0) { av_log(avctx,AV_LOG_ERROR,"Necessary extradata missing!\n"); return -1; } else { /* 8 for mono, 16 for stereo, ? for multichannel Swap to right endianness so we don't need to care later on. */ av_log(avctx,AV_LOG_DEBUG,"codecdata_length=%d\n",avctx->extradata_size); if (avctx->extradata_size >= 8){ q->cookversion = bytestream_get_be32(&edata_ptr); q->samples_per_frame = bytestream_get_be16(&edata_ptr); q->subbands = bytestream_get_be16(&edata_ptr); } if (avctx->extradata_size >= 16){ bytestream_get_be32(&edata_ptr); //Unknown unused q->js_subband_start = bytestream_get_be16(&edata_ptr); q->js_vlc_bits = bytestream_get_be16(&edata_ptr); } } /* Take data from the AVCodecContext (RM container). */ q->sample_rate = avctx->sample_rate; q->nb_channels = avctx->channels; q->bit_rate = avctx->bit_rate; /* Initialize RNG. */ av_init_random(1, &q->random_state); /* Initialize extradata related variables. */ q->samples_per_channel = q->samples_per_frame / q->nb_channels; q->bits_per_subpacket = avctx->block_align * 8; /* Initialize default data states. */ q->log2_numvector_size = 5; q->total_subbands = q->subbands; /* Initialize version-dependent variables */ av_log(NULL,AV_LOG_DEBUG,"q->cookversion=%x\n",q->cookversion); q->joint_stereo = 0; switch (q->cookversion) { case MONO: if (q->nb_channels != 1) { av_log(avctx,AV_LOG_ERROR,"Container channels != 1, report sample!\n"); return -1; } av_log(avctx,AV_LOG_DEBUG,"MONO\n"); break; case STEREO: if (q->nb_channels != 1) { q->bits_per_subpacket = q->bits_per_subpacket/2; } av_log(avctx,AV_LOG_DEBUG,"STEREO\n"); break; case JOINT_STEREO: if (q->nb_channels != 2) { av_log(avctx,AV_LOG_ERROR,"Container channels != 2, report sample!\n"); return -1; } av_log(avctx,AV_LOG_DEBUG,"JOINT_STEREO\n"); if (avctx->extradata_size >= 16){ q->total_subbands = q->subbands + q->js_subband_start; q->joint_stereo = 1; } if (q->samples_per_channel > 256) { q->log2_numvector_size = 6; } if (q->samples_per_channel > 512) { q->log2_numvector_size = 7; } break; case MC_COOK: av_log(avctx,AV_LOG_ERROR,"MC_COOK not supported!\n"); return -1; break; default: av_log(avctx,AV_LOG_ERROR,"Unknown Cook version, report sample!\n"); return -1; break; } /* Initialize variable relations */ q->numvector_size = (1 << q->log2_numvector_size); /* Generate tables */ init_pow2table(); init_gain_table(q); init_cplscales_table(q); if (init_cook_vlc_tables(q) != 0) return -1; if(avctx->block_align >= UINT_MAX/2) return -1; /* Pad the databuffer with: DECODE_BYTES_PAD1 or DECODE_BYTES_PAD2 for decode_bytes(), FF_INPUT_BUFFER_PADDING_SIZE, for the bitstreamreader. */ if (q->nb_channels==2 && q->joint_stereo==0) { q->decoded_bytes_buffer = av_mallocz(avctx->block_align/2 + DECODE_BYTES_PAD2(avctx->block_align/2) + FF_INPUT_BUFFER_PADDING_SIZE); } else { q->decoded_bytes_buffer = av_mallocz(avctx->block_align + DECODE_BYTES_PAD1(avctx->block_align) + FF_INPUT_BUFFER_PADDING_SIZE); } if (q->decoded_bytes_buffer == NULL) return -1; q->gains1.now = q->gain_1; q->gains1.previous = q->gain_2; q->gains2.now = q->gain_3; q->gains2.previous = q->gain_4; /* Initialize transform. */ if ( init_cook_mlt(q) != 0 ) return -1; /* Initialize COOK signal arithmetic handling */ if (1) { q->scalar_dequant = scalar_dequant_float; q->decouple = decouple_float; q->imlt_window = imlt_window_float; q->interpolate = interpolate_float; q->saturate_output = saturate_output_float; } /* Try to catch some obviously faulty streams, othervise it might be exploitable */ if (q->total_subbands > 53) { av_log(avctx,AV_LOG_ERROR,"total_subbands > 53, report sample!\n"); return -1; } if (q->subbands > 50) { av_log(avctx,AV_LOG_ERROR,"subbands > 50, report sample!\n"); return -1; } if ((q->samples_per_channel == 256) || (q->samples_per_channel == 512) || (q->samples_per_channel == 1024)) { } else { av_log(avctx,AV_LOG_ERROR,"unknown amount of samples_per_channel = %d, report sample!\n",q->samples_per_channel); return -1; } if ((q->js_vlc_bits > 6) || (q->js_vlc_bits < 0)) { av_log(avctx,AV_LOG_ERROR,"q->js_vlc_bits = %d, only >= 0 and <= 6 allowed!\n",q->js_vlc_bits); return -1; } avctx->sample_fmt = SAMPLE_FMT_S16; #ifdef COOKDEBUG dump_cook_context(q); #endif return 0; }
static int cook_decode_init(AVCodecContext *avctx) { COOKextradata *e = avctx->extradata; COOKContext *q = avctx->priv_data; /* Take care of the codec specific extradata. */ if (avctx->extradata_size <= 0) { av_log(NULL,AV_LOG_ERROR,"Necessary extradata missing!\n"); return -1; } else { /* 8 for mono, 16 for stereo, ? for multichannel Swap to right endianness so we don't need to care later on. */ av_log(NULL,AV_LOG_DEBUG,"codecdata_length=%d\n",avctx->extradata_size); if (avctx->extradata_size >= 8){ e->cookversion = be2me_32(e->cookversion); e->samples_per_frame = be2me_16(e->samples_per_frame); e->subbands = be2me_16(e->subbands); } if (avctx->extradata_size >= 16){ e->js_subband_start = be2me_16(e->js_subband_start); e->js_vlc_bits = be2me_16(e->js_vlc_bits); } } /* Take data from the AVCodecContext (RM container). */ q->sample_rate = avctx->sample_rate; q->nb_channels = avctx->channels; q->bit_rate = avctx->bit_rate; /* Initialize state. */ q->random_state = 1; /* Initialize extradata related variables. */ q->samples_per_channel = e->samples_per_frame / q->nb_channels; q->samples_per_frame = e->samples_per_frame; q->subbands = e->subbands; q->bits_per_subpacket = avctx->block_align * 8; /* Initialize default data states. */ q->js_subband_start = 0; q->log2_numvector_size = 5; q->total_subbands = q->subbands; /* Initialize version-dependent variables */ av_log(NULL,AV_LOG_DEBUG,"e->cookversion=%x\n",e->cookversion); switch (e->cookversion) { case MONO_COOK1: if (q->nb_channels != 1) { av_log(NULL,AV_LOG_ERROR,"Container channels != 1, report sample!\n"); return -1; } av_log(NULL,AV_LOG_DEBUG,"MONO_COOK1\n"); break; case MONO_COOK2: if (q->nb_channels != 1) { q->joint_stereo = 0; q->bits_per_subpacket = q->bits_per_subpacket/2; } av_log(NULL,AV_LOG_DEBUG,"MONO_COOK2\n"); break; case JOINT_STEREO: if (q->nb_channels != 2) { av_log(NULL,AV_LOG_ERROR,"Container channels != 2, report sample!\n"); return -1; } av_log(NULL,AV_LOG_DEBUG,"JOINT_STEREO\n"); if (avctx->extradata_size >= 16){ q->total_subbands = q->subbands + e->js_subband_start; q->js_subband_start = e->js_subband_start; q->joint_stereo = 1; q->js_vlc_bits = e->js_vlc_bits; } if (q->samples_per_channel > 256) { q->log2_numvector_size = 6; } if (q->samples_per_channel > 512) { q->log2_numvector_size = 7; } break; case MC_COOK: av_log(NULL,AV_LOG_ERROR,"MC_COOK not supported!\n"); return -1; break; default: av_log(NULL,AV_LOG_ERROR,"Unknown Cook version, report sample!\n"); return -1; break; } /* Initialize variable relations */ q->mlt_size = q->samples_per_channel; q->numvector_size = (1 << q->log2_numvector_size); /* Generate tables */ init_rootpow2table(q); init_pow2table(q); init_gain_table(q); if (init_cook_vlc_tables(q) != 0) return -1; /* Pad the databuffer with FF_INPUT_BUFFER_PADDING_SIZE, this is for the bitstreamreader. */ if ((q->decoded_bytes_buffer = av_mallocz((avctx->block_align+(4-avctx->block_align%4) + FF_INPUT_BUFFER_PADDING_SIZE)*sizeof(uint8_t))) == NULL) return -1; q->decode_buf_ptr[0] = q->decode_buffer_1; q->decode_buf_ptr[1] = q->decode_buffer_2; q->decode_buf_ptr[2] = q->decode_buffer_3; q->decode_buf_ptr[3] = q->decode_buffer_4; q->decode_buf_ptr2[0] = q->decode_buffer_3; q->decode_buf_ptr2[1] = q->decode_buffer_4; q->previous_buffer_ptr[0] = q->mono_previous_buffer1; q->previous_buffer_ptr[1] = q->mono_previous_buffer2; /* Initialize transform. */ if ( init_cook_mlt(q) == 0 ) return -1; /* Try to catch some obviously faulty streams, othervise it might be exploitable */ if (q->total_subbands > 53) { av_log(NULL,AV_LOG_ERROR,"total_subbands > 53, report sample!\n"); return -1; } if (q->subbands > 50) { av_log(NULL,AV_LOG_ERROR,"subbands > 50, report sample!\n"); return -1; } if ((q->samples_per_channel == 256) || (q->samples_per_channel == 512) || (q->samples_per_channel == 1024)) { } else { av_log(NULL,AV_LOG_ERROR,"unknown amount of samples_per_channel = %d, report sample!\n",q->samples_per_channel); return -1; } #ifdef COOKDEBUG dump_cook_context(q,e); #endif return 0; }