/* Returns the logical head of the window after we add an image with the give size to its tail.
   Returns NULL when the window is empty, of when we have to empty the window in order
   to insert the new image. */
static WindowImage *glz_dictionary_window_get_new_head(SharedDictionary *dict, int new_image_size)
{
    uint32_t cur_win_size;
    WindowImage *cur_head;

    if ((uint32_t)new_image_size > dict->window.size_limit) {
        dict->cur_usr->error(dict->cur_usr, "image is bigger than window\n");
    }

    GLZ_ASSERT(dict->cur_usr, new_image_size < dict->window.size_limit)

    // the window is empty
    if (!dict->window.used_images_head) {
        return NULL;
    }

    GLZ_ASSERT(dict->cur_usr, dict->window.used_segs_head != NULL_IMAGE_SEG_ID);
    GLZ_ASSERT(dict->cur_usr, dict->window.used_segs_tail != NULL_IMAGE_SEG_ID);

    // used_segs_head is the latest logical head (the physical head may preceed it)
    cur_head = dict->window.segs[dict->window.used_segs_head].image;
    cur_win_size = dict->window.segs[dict->window.used_segs_tail].pixels_num +
        dict->window.segs[dict->window.used_segs_tail].pixels_so_far -
        dict->window.segs[dict->window.used_segs_head].pixels_so_far;

    while ((cur_win_size + new_image_size) > dict->window.size_limit) {
        GLZ_ASSERT(dict->cur_usr, cur_head);
        cur_win_size -= cur_head->size;
        cur_head = cur_head->next;
    }

    return cur_head;
}
void GlzDecoderWindow::add_pre_decoded_image(uint64_t image_id)
{
    GLZ_ASSERT(_debug_calls, image_id > _tail_image_id);
    GLZ_ASSERT(_debug_calls, image_id - _tail_image_id + _n_images < _images_capacity);

    for (uint64_t miss_id = _tail_image_id + 1; miss_id <= image_id; miss_id++) {
        _missing_list.push_back(miss_id);
        _n_images++;
    }
    _tail_image_id = image_id;
}
void glz_enc_dictionary_get_restore_data(GlzEncDictContext *opaque_dict,
                                         GlzEncDictRestoreData *out_data, GlzEncoderUsrContext *usr)
{
    SharedDictionary *dict = (SharedDictionary *)opaque_dict;
    dict->cur_usr = usr;
    GLZ_ASSERT(dict->cur_usr, opaque_dict);
    GLZ_ASSERT(dict->cur_usr, out_data);

    out_data->last_image_id = dict->last_image_id;
    out_data->max_encoders = dict->max_encoders;
    out_data->size = dict->window.size_limit;
}
void GlzDecoderWindow::add_decoded_image(GlzDecodedImage *image)
{
    Lock lock(_new_image_mutex);
    GLZ_ASSERT(_debug_calls, image->get_id() <= _tail_image_id);
    _images[calc_image_win_idx(image->get_id())] = image;
    _new_image_cond.notify_all();
}
void glz_enc_dictionary_reset(GlzEncDictContext *opaque_dict, GlzEncoderUsrContext *usr)
{
    SharedDictionary *dict = (SharedDictionary *)opaque_dict;
    dict->cur_usr = usr;
    GLZ_ASSERT(dict->cur_usr, opaque_dict);

    dict->last_image_id = 0;
    glz_dictionary_window_reset(dict);
    glz_dictionary_reset_hash(dict);
}
/* doesn't call the remove image callback */
void glz_enc_dictionary_remove_image(GlzEncDictContext *opaque_dict,
                                     GlzEncDictImageContext *opaque_image,
                                     GlzEncoderUsrContext *usr)
{
    SharedDictionary *dict = (SharedDictionary *)opaque_dict;
    WindowImage *image = (WindowImage *)opaque_image;
    dict->cur_usr = usr;
    GLZ_ASSERT(dict->cur_usr, opaque_image && opaque_dict);

    glz_dictionary_window_kill_image(dict, image);
}
void glz_dictionary_post_encode(uint32_t encoder_id, GlzEncoderUsrContext *usr,
                                SharedDictionary *dict)
{
    uint32_t i;
    uint32_t early_head_seg = NULL_IMAGE_SEG_ID;
    uint32_t this_encoder_head_seg;

    pthread_rwlock_unlock(&dict->rw_alloc_lock);
    pthread_mutex_lock(&dict->lock);
    dict->cur_usr = usr;

    GLZ_ASSERT(dict->cur_usr, dict->window.encoders_heads[encoder_id] != NULL_IMAGE_SEG_ID);
    // get the earliest head in use (not including this encoder head)
    for (i = 0; i < dict->max_encoders; i++) {
        if (i != encoder_id) {
            if (IMAGE_SEG_IS_EARLIER(dict, dict->window.encoders_heads[i], early_head_seg)) {
                early_head_seg = dict->window.encoders_heads[i];
            }
        }
    }

    // possible only if early_head_seg == NULL
    if (IMAGE_SEG_IS_EARLIER(dict, dict->window.used_segs_head, early_head_seg)) {
        early_head_seg = dict->window.used_segs_head;
    }

    this_encoder_head_seg = dict->window.encoders_heads[encoder_id];

    GLZ_ASSERT(dict->cur_usr, early_head_seg != NULL_IMAGE_SEG_ID);

    if (IMAGE_SEG_IS_EARLIER(dict, this_encoder_head_seg, early_head_seg)) {
        GLZ_ASSERT(dict->cur_usr,
                   this_encoder_head_seg == dict->window.used_images_head->first_seg);
        glz_dictionary_window_remove_head(dict, encoder_id,
                                          dict->window.segs[early_head_seg].image);
    }


    dict->window.encoders_heads[encoder_id] = NULL_IMAGE_SEG_ID;
    pthread_mutex_unlock(&dict->lock);
}
/* Important observations:
   1) When an image is added, if it is not the first missing image, it is not removed
      immediately from the missing list (in order not to store another pointer to
      the missing list inside the window ,or otherwise, to go over the missing list
      and look for the image).
   2) images that weren't removed from the missing list in their addition time, will be removed when
      preliminary images will be added.
   3) The first item in the missing list is always really missing.
   4) The missing list is always ordered by image id (see add_pre_decoded_image)
*/
void GlzDecoderWindow::narrow_window(GlzDecodedImage *last_added)
{
    uint64_t new_head_image_id;

    GLZ_ASSERT(_debug_calls, !_missing_list.empty());

    if (_missing_list.front() != last_added->get_id()) {
        return;
    }

    _missing_list.pop_front();  // removing the last added image from the missing list

    /* maintaining the missing list: removing front images that already arrived */
    while (!_missing_list.empty()) {
        int front_win_idx = calc_image_win_idx(_missing_list.front());
        if (_images[front_win_idx] == NULL) { // still missing
            break;
        } else {
            _missing_list.pop_front();
        }
    }

    /* removing unnecessary image from the window's head*/
    if (_missing_list.empty()) {
        new_head_image_id = _images[
                calc_image_win_idx(_tail_image_id)]->get_window_head_id();
    } else {
        // there must be at least one image in the window since narrow_window is called
        // from post decode
        GLZ_ASSERT(_debug_calls, _images[calc_image_win_idx(_missing_list.front() - 1)]);
        new_head_image_id = _images[
                calc_image_win_idx(_missing_list.front() - 1)]->get_window_head_id();
    }

    remove_head(new_head_image_id);
}
/* moves all the segments that were associated with the images to the free segments */
static inline void __glz_dictionary_window_free_image_segs(SharedDictionary *dict,
                                                           WindowImage *image)
{
    uint32_t old_free_head = dict->window.free_segs_head;
    uint32_t seg_id, next_seg_id;

    GLZ_ASSERT(dict->cur_usr, image->first_seg != NULL_IMAGE_SEG_ID);
    dict->window.free_segs_head = image->first_seg;

    // retrieving the last segment of the image
    for (seg_id = image->first_seg, next_seg_id = dict->window.segs[seg_id].next;
         (next_seg_id != NULL_IMAGE_SEG_ID) && (dict->window.segs[next_seg_id].image == image);
         seg_id = next_seg_id, next_seg_id = dict->window.segs[seg_id].next) {
    }

    // concatenate the free list
    dict->window.segs[seg_id].next = old_free_head;
}
示例#10
0
/* NOTE - it doesn't update the used_segs list*/
static uint32_t __glz_dictionary_window_alloc_image_seg(SharedDictionary *dict)
{
    uint32_t seg_id;
    WindowImageSegment *seg;

    // TODO: when is it best to realloc? when full or when half full?
    if (dict->window.free_segs_head == NULL_IMAGE_SEG_ID) {
        __glz_dictionary_window_segs_realloc(dict);
    }

    GLZ_ASSERT(dict->cur_usr, dict->window.free_segs_head != NULL_IMAGE_SEG_ID);

    seg_id = dict->window.free_segs_head;
    seg = dict->window.segs + seg_id;
    dict->window.free_segs_head = seg->next;

    return seg_id;
}
示例#11
0
WindowImage *glz_dictionary_pre_encode(uint32_t encoder_id, GlzEncoderUsrContext *usr,
                                       SharedDictionary *dict, LzImageType image_type,
                                       int image_width, int image_height, int image_stride,
                                       uint8_t *first_lines, unsigned int num_first_lines,
                                       GlzUsrImageContext *usr_image_context,
                                       uint32_t *image_head_dist)
{
    WindowImage *new_win_head, *ret;
    int image_size;


    pthread_mutex_lock(&dict->lock);

    dict->cur_usr = usr;
    GLZ_ASSERT(dict->cur_usr, dict->window.encoders_heads[encoder_id] == NULL_IMAGE_SEG_ID);

    image_size = __get_pixels_num(image_type, image_height, image_stride);
    new_win_head = glz_dictionary_window_get_new_head(dict, image_size);

    if (!glz_dictionary_is_in_use(dict)) {
        glz_dictionary_window_remove_head(dict, encoder_id, new_win_head);
    }

    ret = glz_dictionary_window_add_image(dict, image_type, image_size, image_height, image_stride,
                                          first_lines, num_first_lines, usr_image_context);

    if (new_win_head) {
        dict->window.encoders_heads[encoder_id] = new_win_head->first_seg;
        *image_head_dist = (uint32_t)(ret->id - new_win_head->id); // shouldn't be greater than 32
                                                                   // bit because the window size is
                                                                   // limited to 2^25
    } else {
        dict->window.encoders_heads[encoder_id] = ret->first_seg;
        *image_head_dist = 0;
    }


    // update encoders head  (the other heads were already updated)
    pthread_mutex_unlock(&dict->lock);
    pthread_rwlock_rdlock(&dict->rw_alloc_lock);
    return ret;
}
inline void GlzDecoderWindow::remove_head(uint64_t new_head_image_id)
{
    GLZ_ASSERT(_debug_calls, _images[_head_idx]);

    int n_images_remove = (int)(new_head_image_id - _images[_head_idx]->get_id());

    if (!n_images_remove) {
        return;
    }

    for (int i = 0; i < n_images_remove; i++) {
        int index = (_head_idx + i) % _images_capacity;
        delete _images[index];
        _images[index] = NULL;
    }

    _n_images -= n_images_remove;
    _head_idx = (_head_idx + n_images_remove) % _images_capacity;

    _release_image_cond.notify_all();
}
示例#13
0
/* compresses one segment starting from 'from'.
   In order to encode a match, we use pixels resolution when we encode RGB image,
   and bytes count when we encode PLT.
*/
static void FNAME(compress_seg)(Encoder *encoder, uint32_t seg_idx, PIXEL *from, int copied)
{
    WindowImageSegment *seg = &encoder->dict->window.segs[seg_idx];
    const PIXEL *ip = from;
    const PIXEL *ip_bound = (PIXEL *)(seg->lines_end) - BOUND_OFFSET;
    const PIXEL *ip_limit = (PIXEL *)(seg->lines_end) - LIMIT_OFFSET;
    int hval;
    int copy = copied;
#ifdef  LZ_PLT
    int pix_per_byte = PLT_PIXELS_PER_BYTE[encoder->cur_image.type];
#endif

#ifdef DEBUG_ENCODE
    int n_encoded = 0;
#endif

    if (copy == 0) {
        encode_copy_count(encoder, MAX_COPY - 1);
    }


    while (LZ_EXPECT_CONDITIONAL(ip < ip_limit)) {
        const PIXEL            *ref;
        const PIXEL            *ref_limit;
        WindowImageSegment     *ref_seg;
        uint32_t ref_seg_idx;
        size_t pix_dist;
        size_t image_dist;
        /* minimum match length */
        size_t len = 0;

        /* comparison starting-point */
        const PIXEL            *anchor = ip;
#ifdef CHAINED_HASH
        int hash_id = 0;
        size_t best_len = 0;
        size_t best_pix_dist = 0;
        size_t best_image_dist = 0;
#endif

        /* check for a run */

        if (LZ_EXPECT_CONDITIONAL(ip > (PIXEL *)(seg->lines))) {
            if (SAME_PIXEL(ip[-1], ip[0]) && SAME_PIXEL(ip[0], ip[1]) && SAME_PIXEL(ip[1], ip[2])) {
                PIXEL x;
                pix_dist = 1;
                image_dist = 0;

                ip += 3;
                ref = anchor + 2;
                ref_limit = (PIXEL *)(seg->lines_end);
                len = 3;

                x = *ref;

                while (ip < ip_bound) { // TODO: maybe separate a run from the same seg or from
                                       // different ones in order to spare ref < ref_limit
                    if (!SAME_PIXEL(*ip, x)) {
                        ip++;
                        break;
                    } else {
                        ip++;
                        len++;
                    }
                }

                goto match;
            } // END RLE MATCH
        }

        /* find potential match */
        HASH_FUNC(hval, ip);

#ifdef CHAINED_HASH
        for (hash_id = 0; hash_id < HASH_CHAIN_SIZE; hash_id++) {
            ref_seg_idx = encoder->dict->htab[hval][hash_id].image_seg_idx;
#else
        ref_seg_idx = encoder->dict->htab[hval].image_seg_idx;
#endif
            ref_seg = encoder->dict->window.segs + ref_seg_idx;
            if (REF_SEG_IS_VALID(encoder->dict, encoder->id,
                                 ref_seg, seg)) {
#ifdef CHAINED_HASH
                ref = ((PIXEL *)ref_seg->lines) + encoder->dict->htab[hval][hash_id].ref_pix_idx;
#else
                ref = ((PIXEL *)ref_seg->lines) + encoder->dict->htab[hval].ref_pix_idx;
#endif
                ref_limit = (PIXEL *)ref_seg->lines_end;

                len = FNAME(do_match)(encoder->dict, ref_seg, ref, ref_limit, seg, ip, ip_bound,
#ifdef  LZ_PLT
                                      pix_per_byte,
#endif
                                      &image_dist, &pix_dist);

#ifdef CHAINED_HASH
                // TODO. not compare len but rather len - encode_size
                if (len > best_len) {
                    best_len = len;
                    best_pix_dist = pix_dist;
                    best_image_dist = image_dist;
                }
#endif
            }

#ifdef CHAINED_HASH
        } // end chain loop
        len = best_len;
        pix_dist = best_pix_dist;
        image_dist = best_image_dist;
#endif

        /* update hash table */
        UPDATE_HASH(encoder->dict, hval, seg_idx, anchor - ((PIXEL *)seg->lines));

        if (!len) {
            goto literal;
        }

match:        // RLE or dictionary (both are encoded by distance from ref (-1) and length)
#ifdef DEBUG_ENCODE
        printf(", match(%d, %d, %d)", image_dist, pix_dist, len);
        n_encoded += len;
#endif

        /* distance is biased */
        if (!image_dist) {
            pix_dist--;
        }

        /* if we have copied something, adjust the copy count */
        if (copy) {
            /* copy is biased, '0' means 1 byte copy */
            update_copy_count(encoder, copy - 1);
        } else {
            /* back, to overwrite the copy count */
            compress_output_prev(encoder);
        }

        /* reset literal counter */
        copy = 0;

        /* length is biased, '1' means a match of 3 pixels for PLT and alpha*/
        /* for RGB 16 1 means 2 */
        /* for RGB24/32 1 means 1...*/
        ip = anchor + len - 2;

#if defined(LZ_RGB16)
        len--;
#elif defined(LZ_PLT) || defined(LZ_RGB_ALPHA)
        len -= 2;
#endif
        GLZ_ASSERT(encoder->usr, len > 0);
        encode_match(encoder, image_dist, pix_dist, len);

        /* update the hash at match boundary */
#if defined(LZ_RGB16) || defined(LZ_RGB24) || defined(LZ_RGB32)
        if (ip > anchor) {
#endif
            HASH_FUNC(hval, ip);
            UPDATE_HASH(encoder->dict, hval, seg_idx, ip - ((PIXEL *)seg->lines));
            ip++;
#if defined(LZ_RGB16) || defined(LZ_RGB24) || defined(LZ_RGB32)
        } else {ip++;
        }
#endif
#if defined(LZ_RGB24) || defined(LZ_RGB32)
        if (ip > anchor) {
#endif
            HASH_FUNC(hval, ip);
            UPDATE_HASH(encoder->dict, hval, seg_idx, ip - ((PIXEL *)seg->lines));
            ip++;
#if defined(LZ_RGB24) || defined(LZ_RGB32)
        } else {
            ip++;
        }
#endif
        /* assuming literal copy */
        encode_copy_count(encoder, MAX_COPY - 1);
        continue;

literal:
#ifdef DEBUG_ENCODE
        printf(", copy");
        n_encoded++;
#endif
        ENCODE_PIXEL(encoder, *anchor);
        anchor++;
        ip = anchor;
        copy++;

        if (LZ_UNEXPECT_CONDITIONAL(copy == MAX_COPY)) {
            copy = 0;
            encode_copy_count(encoder, MAX_COPY - 1);
        }
    } // END LOOP (ip < ip_limit)


    /* left-over as literal copy */
    ip_bound++;
    while (ip <= ip_bound) {
#ifdef DEBUG_ENCODE
        printf(", copy");
        n_encoded++;
#endif
        ENCODE_PIXEL(encoder, *ip);
        ip++;
        copy++;
        if (copy == MAX_COPY) {
            copy = 0;
            encode_copy_count(encoder, MAX_COPY - 1);
        }
    }

    /* if we have copied something, adjust the copy length */
    if (copy) {
        update_copy_count(encoder, copy - 1);
    } else {
        compress_output_prev(encoder);
    }
#ifdef DEBUG_ENCODE
    printf("\ntotal encoded=%d\n", n_encoded);
#endif
}
示例#14
0
void GlzDecoder::decode(uint8_t *data, SpicePalette *palette, void *opaque_usr_info)
{
    DecodedImageWinId image_window_id;
    GlzDecodedImage *decoded_image;
    size_t n_in_bytes_decoded;
    int bytes_per_pixel;
    LzImageType decoded_type;

    _in_start = data;
    _in_now = data;

    decode_header();

#ifdef GLZ_DECODE_TO_RGB32
    bytes_per_pixel = 4;

    if (_image.type == LZ_IMAGE_TYPE_RGBA) {
        decoded_type = LZ_IMAGE_TYPE_RGBA;
    } else {
        decoded_type = LZ_IMAGE_TYPE_RGB32;
    }

#else
    if (IS_IMAGE_TYPE_PLT[_image.type]) {
        GLZ_ASSERT(_debug_calls, !(_image.gross_pixels % PLT_PIXELS_PER_BYTE[_image.type]));
    }
    bytes_per_pixel = RGB_BYTES_PER_PIXEL[_image.type];
    decoded_type = _image.type;
#endif


    image_window_id = _images_window.pre_decode(_image.id, _image.id - _image.win_head_dist);

    decoded_image = _usr_handler.alloc_image(opaque_usr_info, _image.id,
                                             _image.id - _image.win_head_dist,
                                             decoded_type, _image.width, _image.height,
                                             _image.gross_pixels, bytes_per_pixel,
                                             _image.top_down);

    _image.data = decoded_image->get_data();

    // decode_by_type
#ifdef GLZ_DECODE_TO_RGB32
    n_in_bytes_decoded = DECODE_TO_RGB32[_image.type](_images_window, _in_now, _image.data,
                                                      _image.gross_pixels, image_window_id,
                                                      palette, _debug_calls);
#else
    n_in_bytes_decoded = DECODE_TO_SAME[_image.type](_images_window, _in_now, _image.data,
                                                     IS_IMAGE_TYPE_PLT[_image.type] ?
                                                     _image.gross_pixels /
                                                     PLT_PIXELS_PER_BYTE[_image.type] :
                                                     _image.gross_pixels,
                                                     image_window_id, palette, _debug_calls);
#endif

    _in_now += n_in_bytes_decoded;

    if (_image.type == LZ_IMAGE_TYPE_RGBA) {
        glz_rgb_alpha_decode(_images_window, _in_now, _image.data,
                             _image.gross_pixels, image_window_id, palette, _debug_calls);
    }

    _images_window.post_decode(decoded_image);
}