int image_jpeg_load(MediaScanImage *i, MediaScanThumbSpec *spec_hint) {
float scale_factor;
int x, w, h, ofs;
unsigned char *line[1], *ptr = NULL;
JPEGData *j = (JPEGData *)i->_jpeg;
if (setjmp(setjmp_buffer)) {
// See if we have partially decoded an image and hit a fatal error, but still have a usable image
if (ptr != NULL) {
LOG_MEM("destroy JPEG load ptr @ %p\n", ptr);
free(ptr);
ptr = NULL;
}
if (j->cinfo->output_scanline > 0) {
LOG_DEBUG("Fatal error but already processed %d scanlines, continuing...\n", j->cinfo->output_scanline);
return 1;
}
image_jpeg_destroy(i);
return 0;
}
// Abort on progressive JPEGs if memory_limit is in use,
// as progressive JPEGs can use many MBs of memory and there
// is no other easy way to alter libjpeg's memory use
/* XXX
if (i->memory_limit && j->cinfo->progressive_mode) {
LOG_WARN("libmediascan will not decode progressive JPEGs when memory_limit is in use (%s)\n", i->path);
image_jpeg_destroy(i);
return 0;
}
*/
// XXX If reusing the object a second time, we need to read the header again
j->cinfo->do_fancy_upsampling = FALSE;
j->cinfo->do_block_smoothing = FALSE;
// Choose optimal scaling factor
jpeg_calc_output_dimensions(j->cinfo);
scale_factor = (float)j->cinfo->output_width / spec_hint->width;
if (scale_factor > ((float)j->cinfo->output_height / spec_hint->height))
scale_factor = (float)j->cinfo->output_height / spec_hint->height;
if (scale_factor > 1) { // Avoid divide by 0
j->cinfo->scale_denom *= (unsigned int)scale_factor;
jpeg_calc_output_dimensions(j->cinfo);
}
w = j->cinfo->output_width;
h = j->cinfo->output_height;
// Change the original values to the scaled size
i->width = w;
i->height = h;
LOG_DEBUG("Using JPEG scale factor %d/%d, new source dimensions %d x %d\n",
j->cinfo->scale_num, j->cinfo->scale_denom, w, h);
// Save filename in case any warnings/errors occur
strncpy(Filename, i->path, FILENAME_LEN);
if (strlen(i->path) > FILENAME_LEN)
Filename[FILENAME_LEN] = 0;
// Note: I tested libjpeg-turbo's JCS_EXT_XBGR but it writes zeros
// instead of FF for alpha, doesn't support CMYK, etc
jpeg_start_decompress(j->cinfo);
// Allocate storage for decompressed image
image_alloc_pixbuf(i, w, h);
ofs = 0;
ptr = (unsigned char *)malloc(w * j->cinfo->output_components);
line[0] = ptr;
LOG_MEM("new JPEG load ptr @ %p\n", ptr);
if (j->cinfo->output_components == 3) { // RGB
while (j->cinfo->output_scanline < j->cinfo->output_height) {
jpeg_read_scanlines(j->cinfo, line, 1);
for (x = 0; x < w; x++) {
i->_pixbuf[ofs++] = COL(ptr[x + x + x], ptr[x + x + x + 1], ptr[x + x + x + 2]);
}
}
}
else if (j->cinfo->output_components == 4) { // CMYK inverted (Photoshop)
while (j->cinfo->output_scanline < j->cinfo->output_height) {
JSAMPROW row = *line;
jpeg_read_scanlines(j->cinfo, line, 1);
for (x = 0; x < w; x++) {
int c = *row++;
int m = *row++;
int y = *row++;
int k = *row++;
i->_pixbuf[ofs++] = COL((c * k) / MAXJSAMPLE, (m * k) / MAXJSAMPLE, (y * k) / MAXJSAMPLE);
}
}
}
else { // grayscale
while (j->cinfo->output_scanline < j->cinfo->output_height) {
jpeg_read_scanlines(j->cinfo, line, 1);
for (x = 0; x < w; x++) {
i->_pixbuf[ofs++] = COL(ptr[x], ptr[x], ptr[x]);
}
}
}
LOG_MEM("destroy JPEG load ptr @ %p\n", ptr);
free(ptr);
jpeg_finish_decompress(j->cinfo);
return 1;
}
int image_png_load(MediaScanImage *i) {
int bit_depth, color_type, num_passes, x, y;
int ofs;
volatile unsigned char *ptr = NULL; // volatile = won't be rolled back if longjmp is called
PNGData *p = (PNGData *)i->_png;
if (setjmp(png_jmpbuf(p->png_ptr))) {
if (ptr != NULL)
free((void *)ptr);
image_png_destroy(i);
return 0;
}
// XXX If reusing the object a second time, we need to completely create a new png struct
bit_depth = png_get_bit_depth(p->png_ptr, p->info_ptr);
color_type = png_get_color_type(p->png_ptr, p->info_ptr);
if (color_type == PNG_COLOR_TYPE_PALETTE) {
png_set_expand(p->png_ptr); // png_set_palette_to_rgb(p->png_ptr);
i->channels = 4;
}
else if (color_type == PNG_COLOR_TYPE_GRAY && bit_depth < 8)
png_set_expand(p->png_ptr); // png_set_expand_gray_1_2_4_to_8(p->png_ptr);
else if (png_get_valid(p->png_ptr, p->info_ptr, PNG_INFO_tRNS))
png_set_expand(p->png_ptr); // png_set_tRNS_to_alpha(p->png_ptr);
if (bit_depth == 16)
png_set_strip_16(p->png_ptr);
else if (bit_depth < 8)
png_set_packing(p->png_ptr);
// Make non-alpha RGB/Palette 32-bit and Gray 16-bit for easier handling
if (!(color_type & PNG_COLOR_MASK_ALPHA)) {
png_set_add_alpha(p->png_ptr, 0xFF, PNG_FILLER_AFTER);
}
num_passes = png_set_interlace_handling(p->png_ptr);
LOG_DEBUG("png bit_depth %d, color_type %d, channels %d, num_passes %d\n",
bit_depth, color_type, i->channels, num_passes);
png_read_update_info(p->png_ptr, p->info_ptr);
image_alloc_pixbuf(i, i->width, i->height);
ptr = (unsigned char *)malloc(png_get_rowbytes(p->png_ptr, p->info_ptr));
ofs = 0;
if (color_type == PNG_COLOR_TYPE_GRAY || color_type == PNG_COLOR_TYPE_GRAY_ALPHA) { // Grayscale (Alpha)
if (num_passes == 1) { // Non-interlaced
for (y = 0; y < i->height; y++) {
png_read_row(p->png_ptr, (unsigned char *)ptr, NULL);
for (x = 0; x < i->width; x++) {
i->_pixbuf[ofs++] = COL_FULL(ptr[x * 2], ptr[x * 2], ptr[x * 2], ptr[x * 2 + 1]);
}
}
}
else if (num_passes == 7) { // Interlaced
image_png_interlace_pass_gray(i, (unsigned char *)ptr, 0, 8, 0, 8);
image_png_interlace_pass_gray(i, (unsigned char *)ptr, 0, 8, 4, 8);
image_png_interlace_pass_gray(i, (unsigned char *)ptr, 4, 8, 0, 4);
image_png_interlace_pass_gray(i, (unsigned char *)ptr, 0, 4, 2, 4);
image_png_interlace_pass_gray(i, (unsigned char *)ptr, 2, 4, 0, 2);
image_png_interlace_pass_gray(i, (unsigned char *)ptr, 0, 2, 1, 2);
image_png_interlace_pass_gray(i, (unsigned char *)ptr, 1, 2, 0, 1);
}
}
else { // RGB(A)
if (num_passes == 1) { // Non-interlaced
for (y = 0; y < i->height; y++) {
png_read_row(p->png_ptr, (unsigned char *)ptr, NULL);
for (x = 0; x < i->width; x++) {
i->_pixbuf[ofs++] = COL_FULL(ptr[x * 4], ptr[x * 4 + 1], ptr[x * 4 + 2], ptr[x * 4 + 3]);
}
}
}
else if (num_passes == 7) { // Interlaced
// The first pass will return an image 1/8 as wide as the entire image
// (every 8th column starting in column 0)
// and 1/8 as high as the original (every 8th row starting in row 0)
image_png_interlace_pass(i, (unsigned char *)ptr, 0, 8, 0, 8);
// The second will be 1/8 as wide (starting in column 4)
// and 1/8 as high (also starting in row 0)
image_png_interlace_pass(i, (unsigned char *)ptr, 0, 8, 4, 8);
// The third pass will be 1/4 as wide (every 4th pixel starting in column 0)
// and 1/8 as high (every 8th row starting in row 4)
image_png_interlace_pass(i, (unsigned char *)ptr, 4, 8, 0, 4);
// The fourth pass will be 1/4 as wide and 1/4 as high
// (every 4th column starting in column 2, and every 4th row starting in row 0)
image_png_interlace_pass(i, (unsigned char *)ptr, 0, 4, 2, 4);
// The fifth pass will return an image 1/2 as wide,
// and 1/4 as high (starting at column 0 and row 2)
image_png_interlace_pass(i, (unsigned char *)ptr, 2, 4, 0, 2);
// The sixth pass will be 1/2 as wide and 1/2 as high as the original
// (starting in column 1 and row 0)
image_png_interlace_pass(i, (unsigned char *)ptr, 0, 2, 1, 2);
// The seventh pass will be as wide as the original, and 1/2 as high,
// containing all of the odd numbered scanlines.
image_png_interlace_pass(i, (unsigned char *)ptr, 1, 2, 0, 1);
}
else {
FATAL("Unsupported PNG interlace type (%d passes)\n", num_passes);
}
}
free((void *)ptr);
// This is not required, so we can save some time by not reading post-image chunks
//png_read_end(p->png_ptr, p->info_ptr);
return 1;
}
MediaScanImage *video_create_image_from_frame(MediaScanVideo *v, MediaScanResult *r) {
MediaScanImage *i = image_create();
AVFormatContext *avf = (AVFormatContext *)r->_avf;
av_codecs_t *codecs = (av_codecs_t *)v->_codecs;
AVCodec *codec = (AVCodec *)v->_avc;
AVFrame *frame = NULL;
AVPacket packet;
struct SwsContext *swsc = NULL;
int got_picture;
int64_t duration_tb = ((double)avf->duration / AV_TIME_BASE) / av_q2d(codecs->vs->time_base);
uint8_t *src;
int x, y;
int ofs = 0;
int no_keyframe_found = 0;
int skipped_frames = 0;
if ((avcodec_open(codecs->vc, codec)) < 0) {
LOG_ERROR("Couldn't open video codec %s for thumbnail creation\n", codec->name);
goto err;
}
frame = avcodec_alloc_frame();
if (!frame) {
LOG_ERROR("Couldn't allocate a video frame\n");
goto err;
}
av_init_packet(&packet);
i->path = v->path;
i->width = v->width;
i->height = v->height;
// XXX select best video frame, for example:
// * Skip frames of all the same color (e.g. blank intro frames
// * Use edge detection to skip blurry frames
// * http://code.google.com/p/fast-edge/
// * http://en.wikipedia.org/wiki/Canny_edge_detector
// * Use a frame some percentage into the video, what percentage?
// * If really ambitious, use OpenCV for finding a frame with a face?
// XXX other ways to seek if this fails
// XXX for now, seek 10% into the video
av_seek_frame(avf, codecs->vsid, (int)((double)duration_tb * 0.1), 0);
for (;;) {
int ret;
int rgb_bufsize;
AVFrame *frame_rgb = NULL;
uint8_t *rgb_buffer = NULL;
// Give up if we already tried the first frame
if (no_keyframe_found) {
LOG_ERROR("Error decoding video frame for thumbnail: %s\n", v->path);
goto err;
}
if ((ret = av_read_frame(avf, &packet)) < 0) {
if (ret == AVERROR_EOF || skipped_frames > 200) {
LOG_DEBUG("Couldn't find a keyframe, using first frame\n");
no_keyframe_found = 1;
av_seek_frame(avf, codecs->vsid, 0, 0);
av_read_frame(avf, &packet);
}
else {
LOG_ERROR("Couldn't read video frame (%s): ", v->path);
print_averror(ret);
goto err;
}
}
// Skip frame if it's not from the video stream
if (!no_keyframe_found && packet.stream_index != codecs->vsid) {
av_free_packet(&packet);
skipped_frames++;
continue;
}
// Skip non-key-frames
if (!no_keyframe_found && !(packet.flags & AV_PKT_FLAG_KEY)) {
av_free_packet(&packet);
skipped_frames++;
continue;
}
// Skip invalid packets, not sure why this isn't an error from av_read_frame
if (packet.pos < 0) {
av_free_packet(&packet);
skipped_frames++;
continue;
}
LOG_DEBUG("Using video packet: pos %lld size %d, stream_index %d, duration %d\n",
packet.pos, packet.size, packet.stream_index, packet.duration);
if ((ret = avcodec_decode_video2(codecs->vc, frame, &got_picture, &packet)) < 0) {
LOG_ERROR("Error decoding video frame for thumbnail: %s\n", v->path);
print_averror(ret);
goto err;
}
if (!got_picture) {
if (skipped_frames > 200) {
LOG_ERROR("Error decoding video frame for thumbnail: %s\n", v->path);
goto err;
}
if (!no_keyframe_found) {
// Try next frame
av_free_packet(&packet);
skipped_frames++;
continue;
}
}
// use swscale to convert from source format to RGBA in our buffer with no resizing
// XXX what scaler is fastest here when not actually resizing?
swsc = sws_getContext(i->width, i->height, codecs->vc->pix_fmt,
i->width, i->height, PIX_FMT_RGB24, SWS_FAST_BILINEAR, NULL, NULL, NULL);
if (!swsc) {
LOG_ERROR("Unable to get swscale context\n");
goto err;
}
frame_rgb = avcodec_alloc_frame();
if (!frame_rgb) {
LOG_ERROR("Couldn't allocate a video frame\n");
goto err;
}
// XXX There is probably a way to get sws_scale to write directly to i->_pixbuf in our RGBA format
rgb_bufsize = avpicture_get_size(PIX_FMT_RGB24, i->width, i->height);
rgb_buffer = av_malloc(rgb_bufsize);
if (!rgb_buffer) {
LOG_ERROR("Couldn't allocate an RGB video buffer\n");
av_free(frame_rgb);
goto err;
}
LOG_MEM("new rgb_buffer of size %d @ %p\n", rgb_bufsize, rgb_buffer);
avpicture_fill((AVPicture *)frame_rgb, rgb_buffer, PIX_FMT_RGB24, i->width, i->height);
// Convert image to RGB24
sws_scale(swsc, frame->data, frame->linesize, 0, i->height, frame_rgb->data, frame_rgb->linesize);
// Allocate space for our version of the image
image_alloc_pixbuf(i, i->width, i->height);
src = frame_rgb->data[0];
ofs = 0;
for (y = 0; y < i->height; y++) {
for (x = 0; x < i->width * 3; x += 3) {
i->_pixbuf[ofs++] = COL(src[x], src[x + 1], src[x + 2]);
}
src += i->width * 3;
}
// Free the frame
LOG_MEM("destroy rgb_buffer @ %p\n", rgb_buffer);
av_free(rgb_buffer);
av_free(frame_rgb);
// Done!
goto out;
}
err:
image_destroy(i);
i = NULL;
out:
sws_freeContext(swsc);
av_free_packet(&packet);
if (frame)
av_free(frame);
avcodec_close(codecs->vc);
return i;
}
