gboolean fm_folder_model_iter_nth_child(GtkTreeModel *tree_model,
GtkTreeIter *iter,
GtkTreeIter *parent,
gint n)
{
GSequenceIter* items_it;
FmFolderModel* model;
g_return_val_if_fail(FM_IS_FOLDER_MODEL(tree_model), FALSE);
model = FM_FOLDER_MODEL(tree_model);
/* a list has only top-level rows */
if( parent )
return FALSE;
/* special case: if parent == NULL, set iter to n-th top-level row */
if( n >= g_sequence_get_length(model->items) || n < 0 )
return FALSE;
items_it = g_sequence_get_iter_at_pos(model->items, n);
g_assert( items_it != g_sequence_get_end_iter(model->items) );
iter->stamp = model->stamp;
iter->user_data = items_it;
return TRUE;
}
void rclib_lyric_clean(guint index)
{
RCLibLyricParsedData *parsed_data;
RCLibLyricPrivate *priv;
GSequenceIter *begin_iter, *end_iter;
if(lyric_instance==NULL) return;
priv = RCLIB_LYRIC(lyric_instance)->priv;
if(priv==NULL) return;
if(index==1)
parsed_data = &(priv->parsed_data2);
else
parsed_data = &(priv->parsed_data1);
g_free(parsed_data->title);
parsed_data->title = NULL;
g_free(parsed_data->artist);
parsed_data->artist = NULL;
g_free(parsed_data->album);
parsed_data->album = NULL;
g_free(parsed_data->author);
parsed_data->author = NULL;
g_free(parsed_data->filename);
parsed_data->filename = NULL;
parsed_data->offset = 0;
if(parsed_data->seq==NULL) return;
begin_iter = g_sequence_get_begin_iter(parsed_data->seq);
end_iter = g_sequence_get_end_iter(parsed_data->seq);
if(begin_iter==NULL || end_iter==NULL) return;
g_sequence_remove_range(begin_iter, end_iter);
}
void fm_folder_model_file_deleted(FmFolderModel* model, FmFileInfo* file)
{
GSequenceIter *seq_it;
/* not required for hidden files */
gboolean update_view;
#if 0
/* If there is no file info, that means the dir itself was deleted. */
if( G_UNLIKELY(!file) )
{
/* Clear the whole list */
GSequenceIter *items_it = g_sequence_get_begin_iter(model->items);
path = gtk_tree_path_new_from_indices(0, -1);
while( !g_sequence_iter_is_end(items_it) )
{
gtk_tree_model_row_deleted(GTK_TREE_MODEL(model), path);
file = (VFSFileInfo*)g_sequence_get(items_it);
items_it = g_sequence_iter_next(it);
vfs_file_info_unref(file);
}
for( l = model->items; l; l = model->items )
{
gtk_tree_model_row_deleted(GTK_TREE_MODEL(model), path);
file = (VFSFileInfo*)l->data;
model->items = g_list_delete_link(model->items, l);
vfs_file_info_unref(file);
}
g_sequence_remove_range( g_sequence_get_begin_iter(model->items), g_sequence_get_end_iter(model->items) );
gtk_tree_path_free(path);
return;
}
#endif
if( !model->show_hidden && fm_file_info_is_hidden(file) ) /* if this is a hidden file */
{
update_view = FALSE;
seq_it = g_sequence_get_begin_iter(model->hidden);
}
else
{
update_view = TRUE;
seq_it = g_sequence_get_begin_iter(model->items);
}
while( !g_sequence_iter_is_end(seq_it) )
{
FmFolderItem* item = (FmFolderItem*)g_sequence_get(seq_it);
if( item->inf == file )
break;
seq_it = g_sequence_iter_next(seq_it);
}
if( update_view )
{
GtkTreePath* path = gtk_tree_path_new_from_indices(g_sequence_iter_get_position(seq_it), -1);
gtk_tree_model_row_deleted(GTK_TREE_MODEL(model), path);
gtk_tree_path_free(path);
}
model->iter_age++;
g_sequence_remove(seq_it);
}
gboolean fm_folder_model_get_iter(GtkTreeModel *tree_model,
GtkTreeIter *iter,
GtkTreePath *path)
{
FmFolderModel* model;
gint *indices, n, depth;
GSequenceIter* items_it;
g_assert( FM_IS_FOLDER_MODEL(tree_model) );
g_assert(path!=NULL);
model = FM_FOLDER_MODEL(tree_model);
indices = gtk_tree_path_get_indices(path);
depth = gtk_tree_path_get_depth(path);
/* we do not allow children */
g_assert(depth == 1); /* depth 1 = top level; a list only has top level nodes and no children */
n = indices[0]; /* the n-th top level row */
if( n >= g_sequence_get_length(model->items) || n < 0 )
return FALSE;
items_it = g_sequence_get_iter_at_pos(model->items, n);
g_assert( items_it != g_sequence_get_end_iter(model->items) );
/* We simply store a pointer in the iter */
iter->stamp = model->stamp;
iter->user_data = items_it;
return TRUE;
}
static void
byzanz_recorder_dispose (GObject *object)
{
ByzanzRecorder *recorder = BYZANZ_RECORDER (object);
g_sequence_remove_range (g_sequence_get_begin_iter (recorder->layers),
g_sequence_get_end_iter (recorder->layers));
G_OBJECT_CLASS (byzanz_recorder_parent_class)->dispose (object);
}
static void
fm_ogl_model_clear_directory (FMOGLModel *model, GSequence *files)
{
GSequenceIter *iter = g_sequence_get_begin_iter(files);
while( iter != g_sequence_get_end_iter(files)){
GSequenceIter *iter_next = g_sequence_iter_next(iter);
fm_ogl_model_remove(model, iter);
iter = iter_next;
}
}
void free_segment(struct segment* s) {
GSequenceIter *begin = g_sequence_get_begin_iter(s->chunks);
GSequenceIter *end = g_sequence_get_end_iter(s->chunks);
for(; begin != end; begin = g_sequence_get_begin_iter(s->chunks)){
free_chunk(g_sequence_get(begin));
g_sequence_remove(begin);
}
g_sequence_free(s->chunks);
if (s->features)
g_hash_table_destroy(s->features);
free(s);
}
void
swfdec_text_buffer_insert_text (SwfdecTextBuffer *buffer,
gsize pos, const char *text)
{
gsize len;
GSequenceIter *iter;
SwfdecTextBufferFormat *format;
g_return_if_fail (SWFDEC_IS_TEXT_BUFFER (buffer));
g_return_if_fail (pos <= buffer->text->len);
g_return_if_fail (text != NULL);
len = strlen (text);
if (len == 0)
return;
if (pos == buffer->text->len) {
g_string_insert_len (buffer->text, pos, text, len);
format = swfdec_text_buffer_format_new ();
format->start = pos;
swfdec_text_attributes_copy (&format->attr,
&buffer->default_attributes, SWFDEC_TEXT_ATTRIBUTES_MASK);
iter = g_sequence_append (buffer->attributes, format);
swfdec_text_buffer_remove_duplicates (g_sequence_iter_prev (iter),
g_sequence_iter_next (iter));
} else {
g_string_insert_len (buffer->text, pos, text, len);
iter = g_sequence_get_end_iter (buffer->attributes);
for (;;) {
iter = g_sequence_iter_prev (iter);
format = g_sequence_get (iter);
if (format->start <= pos)
break;
format->start += len;
}
}
CHECK_ATTRIBUTES (buffer);
/* adapt cursor */
if (buffer->cursor_start >= pos)
buffer->cursor_start += len;
if (buffer->cursor_end >= pos)
buffer->cursor_end += len;
g_signal_emit (buffer, signals[TEXT_CHANGED], 0);
g_signal_emit (buffer, signals[CURSOR_CHANGED], 0,
(gulong) MIN (buffer->cursor_start, buffer->cursor_end),
(gulong) MAX (buffer->cursor_start, buffer->cursor_end));
}
void
swfdec_text_buffer_set_attributes (SwfdecTextBuffer *buffer, gsize start,
gsize length, const SwfdecTextAttributes *attr, guint mask)
{
SwfdecTextBufferFormat *format;
GSequenceIter *start_iter, *iter, *end_iter;
gsize end;
g_return_if_fail (SWFDEC_IS_TEXT_BUFFER (buffer));
g_return_if_fail (start + length <= buffer->text->len);
g_return_if_fail (length > 0);
g_return_if_fail (attr != NULL);
g_return_if_fail (mask != 0);
/* ensure start and end position have an attribtues entry. If not, create one */
end = start + length;
start_iter = swfdec_text_buffer_get_iter_for_pos (buffer, start);
format = g_sequence_get (start_iter);
if (format->start < start) {
start_iter = swfdec_text_buffer_copy_format (buffer, start_iter, start);
}
if (end == buffer->text->len) {
end_iter = g_sequence_get_end_iter (buffer->attributes);
} else {
end_iter = swfdec_text_buffer_get_iter_for_pos (buffer, end);
format = g_sequence_get (end_iter);
if (format->start < end) {
end_iter = swfdec_text_buffer_copy_format (buffer, end_iter, end);
}
}
/* start_iter points to first item to modify, end_iter points after last item to modify */
/* modify all formats in range [start_iter, end_iter) */
for (iter = start_iter; iter != end_iter; iter = g_sequence_iter_next (iter)) {
format = g_sequence_get (iter);
swfdec_text_attributes_copy (&format->attr, attr, mask);
}
/* remove entries that are identical now */
swfdec_text_buffer_remove_duplicates (g_sequence_iter_prev (start_iter),
g_sequence_iter_next (end_iter));
CHECK_ATTRIBUTES (buffer);
g_signal_emit (buffer, signals[TEXT_CHANGED], 0);
}
static gboolean
clutter_list_model_iter_is_last (ClutterModelIter *iter)
{
ClutterListModelIter *iter_default;
ClutterModelIter *temp_iter;
ClutterModel *model;
GSequence *sequence;
GSequenceIter *begin, *end;
iter_default = CLUTTER_LIST_MODEL_ITER (iter);
g_assert (iter_default->seq_iter != NULL);
if (g_sequence_iter_is_end (iter_default->seq_iter))
return TRUE;
model = clutter_model_iter_get_model (iter);
sequence = CLUTTER_LIST_MODEL (model)->priv->sequence;
begin = g_sequence_get_end_iter (sequence);
begin = g_sequence_iter_prev (begin);
end = iter_default->seq_iter;
temp_iter = CLUTTER_LIST_MODEL (model)->priv->temp_iter;
while (!g_sequence_iter_is_begin (begin))
{
CLUTTER_LIST_MODEL_ITER (temp_iter)->seq_iter = begin;
if (clutter_model_filter_iter (model, temp_iter))
{
end = begin;
break;
}
begin = g_sequence_iter_prev (begin);
}
/* This is because the 'end_iter' is always *after* the last valid iter.
* Otherwise we'd have endless loops
*/
end = g_sequence_iter_next (end);
return iter_default->seq_iter == end;
}
GSequenceIter *rclib_lyric_get_line_iter(guint index, gint64 time)
{
RCLibLyricParsedData *parsed_data;
RCLibLyricPrivate *priv;
RCLibLyricData *lyric_data = NULL;
GSequenceIter *begin, *end, *iter;
gint64 offset;
if(lyric_instance==NULL) return NULL;
priv = RCLIB_LYRIC(lyric_instance)->priv;
if(priv==NULL) return NULL;
if(index==1)
parsed_data = &(priv->parsed_data2);
else
parsed_data = &(priv->parsed_data1);
begin = g_sequence_get_begin_iter(parsed_data->seq);
end = g_sequence_get_end_iter(parsed_data->seq);
if(begin==end) return NULL;
lyric_data = g_sequence_get(begin);
offset = (gint64)parsed_data->offset * GST_MSECOND;
if(lyric_data!=NULL)
{
if(time<lyric_data->time+offset) return NULL;
}
do
{
iter = g_sequence_range_get_midpoint(begin, end);
lyric_data = g_sequence_get(iter);
if(lyric_data==NULL) break;
if(time<lyric_data->time+offset)
{
end = iter;
}
else if(time>=lyric_data->time+offset && (lyric_data->length<0 ||
time<lyric_data->time+offset+lyric_data->length))
{
break;
}
else begin = iter;
}
while(begin!=end);
return iter;
}
/* Determine the full extent of a rangeset--the smallest offset covered and the
* length needed to extend to the end of the last item. */
void bluesky_rangeset_get_extents(BlueSkyRangeset *rangeset,
uint64_t *start, uint64_t *length)
{
GSequenceIter *i;
BlueSkyRangesetItem *item;
i = g_sequence_get_begin_iter(rangeset->seq);
if (g_sequence_iter_is_end(i)) {
*start = 0;
*length = 0;
return;
}
item = (BlueSkyRangesetItem *)g_sequence_get(i);
*start = item->start;
i = g_sequence_get_end_iter(rangeset->seq);
i = g_sequence_iter_prev(i);
item = (BlueSkyRangesetItem *)g_sequence_get(i);
*length = (item->start + item->length) - *start;
}
void send_segment(struct segment* s) {
/*
* CHUNK_SEGMENT_START and _END are used for
* reconstructing the segment in filter phase.
*/
struct chunk* ss = new_chunk(0);
SET_CHUNK(ss, CHUNK_SEGMENT_START);
sync_queue_push(dedup_queue, ss);
GSequenceIter *end = g_sequence_get_end_iter(s->chunks);
GSequenceIter *begin = g_sequence_get_begin_iter(s->chunks);
while(begin != end) {
struct chunk* c = g_sequence_get(begin);
if (!CHECK_CHUNK(c, CHUNK_FILE_START) && !CHECK_CHUNK(c, CHUNK_FILE_END)) {
if (CHECK_CHUNK(c, CHUNK_DUPLICATE)) {
if (c->id == TEMPORARY_ID) {
DEBUG("Dedup phase: %ldth chunk is identical to a unique chunk",
chunk_num++);
} else {
DEBUG("Dedup phase: %ldth chunk is duplicate in container %lld",
chunk_num++, c->id);
}
} else {
DEBUG("Dedup phase: %ldth chunk is unique", chunk_num++);
}
}
sync_queue_push(dedup_queue, c);
g_sequence_remove(begin);
begin = g_sequence_get_begin_iter(s->chunks);
}
struct chunk* se = new_chunk(0);
SET_CHUNK(se, CHUNK_SEGMENT_END);
sync_queue_push(dedup_queue, se);
s->chunk_num = 0;
}
void fm_folder_model_file_changed(FmFolderModel* model, FmFileInfo* file)
{
FmFolderItem* item;
GSequenceIter* items_it;
GtkTreeIter it;
GtkTreePath* path;
if( !model->show_hidden && fm_file_info_is_hidden(file) )
return;
items_it = g_sequence_get_begin_iter(model->items);
/* FIXME: write a GCompareDataFunc for this */
while( !g_sequence_iter_is_end(items_it) )
{
item = (FmFolderItem*)g_sequence_get(items_it);
if( item->inf == file )
break;
items_it = g_sequence_iter_next(items_it);
}
if( items_it == g_sequence_get_end_iter(model->items) )
return;
/* update the icon */
if( item->icon )
{
g_object_unref(item->icon);
item->icon = NULL;
}
it.stamp = model->stamp;
it.user_data = items_it;
path = gtk_tree_path_new_from_indices(g_sequence_iter_get_position(items_it), -1);
gtk_tree_model_row_changed(GTK_TREE_MODEL(model), path, &it);
gtk_tree_path_free(path);
}
void index_lookup_similarity_detection(struct segment *s){
assert(s->features);
top_segment_select(s->features);
GSequenceIter *iter = g_sequence_get_begin_iter(s->chunks);
GSequenceIter *end = g_sequence_get_end_iter(s->chunks);
for (; iter != end; iter = g_sequence_iter_next(iter)) {
struct chunk* c = g_sequence_get(iter);
if (CHECK_CHUNK(c, CHUNK_FILE_START) || CHECK_CHUNK(c, CHUNK_FILE_END))
continue;
/* First check it in the storage buffer */
if(storage_buffer.container_buffer
&& lookup_fingerprint_in_container(storage_buffer.container_buffer, &c->fp)){
c->id = get_container_id(storage_buffer.container_buffer);
SET_CHUNK(c, CHUNK_DUPLICATE);
SET_CHUNK(c, CHUNK_REWRITE_DENIED);
}
/*
* First check the buffered fingerprints,
* recently backup fingerprints.
*/
GQueue *tq = g_hash_table_lookup(index_buffer.buffered_fingerprints, &c->fp);
if (!tq) {
tq = g_queue_new();
} else if (!CHECK_CHUNK(c, CHUNK_DUPLICATE)) {
struct indexElem *be = g_queue_peek_head(tq);
c->id = be->id;
SET_CHUNK(c, CHUNK_DUPLICATE);
}
/* Check the fingerprint cache */
if (!CHECK_CHUNK(c, CHUNK_DUPLICATE)) {
/* Searching in fingerprint cache */
int64_t id = fingerprint_cache_lookup(&c->fp);
if(id != TEMPORARY_ID){
c->id = id;
SET_CHUNK(c, CHUNK_DUPLICATE);
}
}
if(destor.index_category[0] == INDEX_CATEGORY_EXACT
|| destor.index_segment_selection_method[0] == INDEX_SEGMENT_SELECT_MIX){
if (!CHECK_CHUNK(c, CHUNK_DUPLICATE)) {
/* Searching in key-value store */
int64_t* ids = kvstore_lookup((char*)&c->fp);
if(ids){
index_overhead.lookup_requests++;
/* prefetch the target unit */
fingerprint_cache_prefetch(ids[0]);
int64_t id = fingerprint_cache_lookup(&c->fp);
if(id != TEMPORARY_ID){
/*
* It can be not cached,
* since a partial key is possible in near-exact deduplication.
*/
c->id = id;
SET_CHUNK(c, CHUNK_DUPLICATE);
}else{
NOTICE("Filter phase: A key collision occurs");
}
}else{
index_overhead.lookup_requests_for_unique++;
VERBOSE("Dedup phase: non-existing fingerprint");
}
}
}
/* Insert it into the index buffer */
struct indexElem *ne = (struct indexElem*) malloc(
sizeof(struct indexElem));
ne->id = c->id;
memcpy(&ne->fp, &c->fp, sizeof(fingerprint));
g_queue_push_tail(tq, ne);
g_hash_table_replace(index_buffer.buffered_fingerprints, &ne->fp, tq);
index_buffer.chunk_num++;
}
}
/*
* When a container buffer is full, we push it into container_queue.
*/
static void* filter_thread(void *arg) {
int enable_rewrite = 1;
struct fileRecipeMeta* r = NULL;
while (1) {
struct chunk* c = sync_queue_pop(rewrite_queue);
if (c == NULL)
/* backup job finish */
break;
/* reconstruct a segment */
struct segment* s = new_segment();
/* segment head */
assert(CHECK_CHUNK(c, CHUNK_SEGMENT_START));
free_chunk(c);
c = sync_queue_pop(rewrite_queue);
while (!(CHECK_CHUNK(c, CHUNK_SEGMENT_END))) {
g_sequence_append(s->chunks, c);
if (!CHECK_CHUNK(c, CHUNK_FILE_START)
&& !CHECK_CHUNK(c, CHUNK_FILE_END))
s->chunk_num++;
c = sync_queue_pop(rewrite_queue);
}
free_chunk(c);
/* For self-references in a segment.
* If we find an early copy of the chunk in this segment has been rewritten,
* the rewrite request for it will be denied to avoid repeat rewriting. */
GHashTable *recently_rewritten_chunks = g_hash_table_new_full(g_int64_hash,
g_fingerprint_equal, NULL, free_chunk);
GHashTable *recently_unique_chunks = g_hash_table_new_full(g_int64_hash,
g_fingerprint_equal, NULL, free_chunk);
pthread_mutex_lock(&index_lock.mutex);
TIMER_DECLARE(1);
TIMER_BEGIN(1);
/* This function will check the fragmented chunks
* that would be rewritten later.
* If we find an early copy of the chunk in earlier segments,
* has been rewritten,
* the rewrite request for it will be denied. */
index_check_buffer(s);
GSequenceIter *iter = g_sequence_get_begin_iter(s->chunks);
GSequenceIter *end = g_sequence_get_end_iter(s->chunks);
for (; iter != end; iter = g_sequence_iter_next(iter)) {
c = g_sequence_get(iter);
if (CHECK_CHUNK(c, CHUNK_FILE_START) || CHECK_CHUNK(c, CHUNK_FILE_END))
continue;
VERBOSE("Filter phase: %dth chunk in %s container %lld", chunk_num,
CHECK_CHUNK(c, CHUNK_OUT_OF_ORDER) ? "out-of-order" : "", c->id);
/* Cache-Aware Filter */
if (destor.rewrite_enable_cache_aware && restore_aware_contains(c->id)) {
assert(c->id != TEMPORARY_ID);
VERBOSE("Filter phase: %dth chunk is cached", chunk_num);
SET_CHUNK(c, CHUNK_IN_CACHE);
}
/* A cfl-switch for rewriting out-of-order chunks. */
if (destor.rewrite_enable_cfl_switch) {
double cfl = restore_aware_get_cfl();
if (enable_rewrite && cfl > destor.rewrite_cfl_require) {
VERBOSE("Filter phase: Turn OFF the (out-of-order) rewrite switch of %.3f",
cfl);
enable_rewrite = 0;
} else if (!enable_rewrite && cfl < destor.rewrite_cfl_require) {
VERBOSE("Filter phase: Turn ON the (out-of-order) rewrite switch of %.3f",
cfl);
enable_rewrite = 1;
}
}
if(CHECK_CHUNK(c, CHUNK_DUPLICATE) && c->id == TEMPORARY_ID){
struct chunk* ruc = g_hash_table_lookup(recently_unique_chunks, &c->fp);
assert(ruc);
c->id = ruc->id;
}
struct chunk* rwc = g_hash_table_lookup(recently_rewritten_chunks, &c->fp);
if(rwc){
c->id = rwc->id;
SET_CHUNK(c, CHUNK_REWRITE_DENIED);
}
/* A fragmented chunk will be denied if it has been rewritten recently */
if (!CHECK_CHUNK(c, CHUNK_DUPLICATE) || (!CHECK_CHUNK(c, CHUNK_REWRITE_DENIED)
&& (CHECK_CHUNK(c, CHUNK_SPARSE)
|| (enable_rewrite && CHECK_CHUNK(c, CHUNK_OUT_OF_ORDER)
&& !CHECK_CHUNK(c, CHUNK_IN_CACHE))))) {
/*
* If the chunk is unique, or be fragmented and not denied,
* we write it to a container.
* Fragmented indicates: sparse, or out of order and not in cache,
*/
if (storage_buffer.container_buffer == NULL){
storage_buffer.container_buffer = create_container();
if(destor.index_category[1] == INDEX_CATEGORY_PHYSICAL_LOCALITY)
storage_buffer.chunks = g_sequence_new(free_chunk);
}
if (container_overflow(storage_buffer.container_buffer, c->size)) {
if(destor.index_category[1] == INDEX_CATEGORY_PHYSICAL_LOCALITY){
/*
* TO-DO
* Update_index for physical locality
*/
GHashTable *features = sampling(storage_buffer.chunks,
g_sequence_get_length(storage_buffer.chunks));
index_update(features, get_container_id(storage_buffer.container_buffer));
g_hash_table_destroy(features);
g_sequence_free(storage_buffer.chunks);
storage_buffer.chunks = g_sequence_new(free_chunk);
}
TIMER_END(1, jcr.filter_time);
write_container_async(storage_buffer.container_buffer);
TIMER_BEGIN(1);
storage_buffer.container_buffer = create_container();
}
if(add_chunk_to_container(storage_buffer.container_buffer, c)){
struct chunk* wc = new_chunk(0);
memcpy(&wc->fp, &c->fp, sizeof(fingerprint));
wc->id = c->id;
if (!CHECK_CHUNK(c, CHUNK_DUPLICATE)) {
jcr.unique_chunk_num++;
jcr.unique_data_size += c->size;
g_hash_table_insert(recently_unique_chunks, &wc->fp, wc);
VERBOSE("Filter phase: %dth chunk is recently unique, size %d", chunk_num,
g_hash_table_size(recently_unique_chunks));
} else {
jcr.rewritten_chunk_num++;
jcr.rewritten_chunk_size += c->size;
g_hash_table_insert(recently_rewritten_chunks, &wc->fp, wc);
}
if(destor.index_category[1] == INDEX_CATEGORY_PHYSICAL_LOCALITY){
struct chunk* ck = new_chunk(0);
memcpy(&ck->fp, &c->fp, sizeof(fingerprint));
g_sequence_append(storage_buffer.chunks, ck);
}
VERBOSE("Filter phase: Write %dth chunk to container %lld",
chunk_num, c->id);
}else{
VERBOSE("Filter phase: container %lld already has this chunk", c->id);
assert(destor.index_category[0] != INDEX_CATEGORY_EXACT
|| destor.rewrite_algorithm[0]!=REWRITE_NO);
}
}else{
if(CHECK_CHUNK(c, CHUNK_REWRITE_DENIED)){
VERBOSE("Filter phase: %lldth fragmented chunk is denied", chunk_num);
}else if (CHECK_CHUNK(c, CHUNK_OUT_OF_ORDER)) {
VERBOSE("Filter phase: %lldth chunk in out-of-order container %lld is already cached",
chunk_num, c->id);
}
}
assert(c->id != TEMPORARY_ID);
/* Collect historical information. */
har_monitor_update(c->id, c->size);
/* Restore-aware */
restore_aware_update(c->id, c->size);
chunk_num++;
}
int full = index_update_buffer(s);
/* Write a SEGMENT_BEGIN */
segmentid sid = append_segment_flag(jcr.bv, CHUNK_SEGMENT_START, s->chunk_num);
/* Write recipe */
iter = g_sequence_get_begin_iter(s->chunks);
end = g_sequence_get_end_iter(s->chunks);
for (; iter != end; iter = g_sequence_iter_next(iter)) {
c = g_sequence_get(iter);
if(r == NULL){
assert(CHECK_CHUNK(c,CHUNK_FILE_START));
r = new_file_recipe_meta(c->data);
}else if(!CHECK_CHUNK(c,CHUNK_FILE_END)){
struct chunkPointer cp;
cp.id = c->id;
assert(cp.id>=0);
memcpy(&cp.fp, &c->fp, sizeof(fingerprint));
cp.size = c->size;
append_n_chunk_pointers(jcr.bv, &cp ,1);
r->chunknum++;
r->filesize += c->size;
jcr.chunk_num++;
jcr.data_size += c->size;
}else{
assert(CHECK_CHUNK(c,CHUNK_FILE_END));
append_file_recipe_meta(jcr.bv, r);
free_file_recipe_meta(r);
r = NULL;
jcr.file_num++;
}
}
/* Write a SEGMENT_END */
append_segment_flag(jcr.bv, CHUNK_SEGMENT_END, 0);
if(destor.index_category[1] == INDEX_CATEGORY_LOGICAL_LOCALITY){
/*
* TO-DO
* Update_index for logical locality
*/
s->features = sampling(s->chunks, s->chunk_num);
if(destor.index_category[0] == INDEX_CATEGORY_EXACT){
/*
* For exact deduplication,
* unique fingerprints are inserted.
*/
VERBOSE("Filter phase: add %d unique fingerprints to %d features",
g_hash_table_size(recently_unique_chunks),
g_hash_table_size(s->features));
GHashTableIter iter;
gpointer key, value;
g_hash_table_iter_init(&iter, recently_unique_chunks);
while(g_hash_table_iter_next(&iter, &key, &value)){
struct chunk* uc = value;
fingerprint *ft = malloc(sizeof(fingerprint));
memcpy(ft, &uc->fp, sizeof(fingerprint));
g_hash_table_insert(s->features, ft, NULL);
}
/*
* OPTION:
* It is still an open problem whether we need to update
* rewritten fingerprints.
* It would increase index update overhead, while the benefit
* remains unclear.
* More experiments are required.
*/
VERBOSE("Filter phase: add %d rewritten fingerprints to %d features",
g_hash_table_size(recently_rewritten_chunks),
g_hash_table_size(s->features));
g_hash_table_iter_init(&iter, recently_rewritten_chunks);
while(g_hash_table_iter_next(&iter, &key, &value)){
struct chunk* uc = value;
fingerprint *ft = malloc(sizeof(fingerprint));
memcpy(ft, &uc->fp, sizeof(fingerprint));
g_hash_table_insert(s->features, ft, NULL);
}
}
index_update(s->features, sid);
}
free_segment(s);
if(index_lock.wait_threshold > 0 && full == 0){
pthread_cond_broadcast(&index_lock.cond);
}
TIMER_END(1, jcr.filter_time);
pthread_mutex_unlock(&index_lock.mutex);
g_hash_table_destroy(recently_rewritten_chunks);
g_hash_table_destroy(recently_unique_chunks);
}
if (storage_buffer.container_buffer
&& !container_empty(storage_buffer.container_buffer)){
if(destor.index_category[1] == INDEX_CATEGORY_PHYSICAL_LOCALITY){
/*
* TO-DO
* Update_index for physical locality
*/
GHashTable *features = sampling(storage_buffer.chunks,
g_sequence_get_length(storage_buffer.chunks));
index_update(features, get_container_id(storage_buffer.container_buffer));
g_hash_table_destroy(features);
g_sequence_free(storage_buffer.chunks);
}
write_container_async(storage_buffer.container_buffer);
}
/* All files done */
jcr.status = JCR_STATUS_DONE;
return NULL;
}
gboolean rclib_lyric_load_file(const gchar *filename, guint index)
{
RCLibLyricParsedData *parsed_data;
gchar *line;
GFile *file;
GFileInputStream *input_stream;
GDataInputStream *data_stream;
gsize line_len;
gint64 time = -1;
RCLibLyricPrivate *priv;
GSequenceIter *iter;
RCLibLyricData *lyric_data;
rclib_lyric_clean(index);
if(lyric_instance==NULL) return FALSE;
if(filename==NULL) return FALSE;
priv = RCLIB_LYRIC(lyric_instance)->priv;
if(priv==NULL) return FALSE;
file = g_file_new_for_path(filename);
if(file==NULL) return FALSE;
if(!g_file_query_exists(file, NULL))
{
g_object_unref(file);
return FALSE;
}
input_stream = g_file_read(file, NULL, NULL);
g_object_unref(file);
if(input_stream==NULL) return FALSE;
data_stream = g_data_input_stream_new(G_INPUT_STREAM(input_stream));
g_object_unref(input_stream);
if(data_stream==NULL) return FALSE;
g_data_input_stream_set_newline_type(data_stream,
G_DATA_STREAM_NEWLINE_TYPE_ANY);
if(index==1)
parsed_data = &(priv->parsed_data2);
else
parsed_data = &(priv->parsed_data1);
if(index!=1) index = 0;
while((line=g_data_input_stream_read_line(data_stream, &line_len,
NULL, NULL))!=NULL)
{
rclib_lyric_add_line(parsed_data, priv->regex, priv->encoding,
line, line_len);
g_free(line);
}
g_object_unref(data_stream);
for(iter = g_sequence_get_end_iter(parsed_data->seq);
!g_sequence_iter_is_begin(iter);)
{
iter = g_sequence_iter_prev(iter);
lyric_data = g_sequence_get(iter);
if(lyric_data==NULL) continue;
if(time>=0)
{
lyric_data->length = time - lyric_data->time;
}
time = lyric_data->time;
}
parsed_data->filename = g_strdup(filename);
g_signal_emit(lyric_instance, lyric_signals[SIGNAL_LYRIC_READY], 0,
index);
return TRUE;
}
static ClutterModelIter *
clutter_list_model_get_iter_at_row (ClutterModel *model,
guint row)
{
ClutterListModel *model_default = CLUTTER_LIST_MODEL (model);
GSequence *sequence = model_default->priv->sequence;
gint seq_length = g_sequence_get_length (sequence);
ClutterListModelIter *retval;
if (row >= seq_length)
return NULL;
retval = g_object_new (CLUTTER_TYPE_LIST_MODEL_ITER,
"model", model,
"row", row,
NULL);
/* short-circuit in case we don't have a filter in place */
if (!clutter_model_get_filter_set (model))
{
retval->seq_iter = g_sequence_get_iter_at_pos (sequence, row);
return CLUTTER_MODEL_ITER (retval);
}
if (row == 0)
{
GSequenceIter *filter_next;
gboolean row_found = FALSE;
filter_next = g_sequence_get_begin_iter (sequence);
g_assert (filter_next != NULL);
while (!g_sequence_iter_is_end (filter_next))
{
retval->seq_iter = filter_next;
if (clutter_model_filter_iter (model, CLUTTER_MODEL_ITER (retval)))
{
/* We've found a row that is valid under the filter */
row_found = TRUE;
break;
}
filter_next = g_sequence_iter_next (filter_next);
}
/* Everything has been filtered -> there is no first row */
if (!row_found)
{
g_object_unref (retval);
return NULL;
}
}
else
{
GSequenceIter *filter_prev;
filter_prev = g_sequence_get_end_iter (sequence);
g_assert (filter_prev != NULL);
filter_prev = g_sequence_iter_prev (filter_prev);
while (!g_sequence_iter_is_begin (filter_prev))
{
retval->seq_iter = filter_prev;
if (clutter_model_filter_iter (model, CLUTTER_MODEL_ITER (retval)))
break;
filter_prev = g_sequence_iter_prev (filter_prev);
}
}
return CLUTTER_MODEL_ITER (retval);
}
void close_har() {
sds fname = sdsdup(destor.working_directory);
fname = sdscat(fname, "recipes/bv");
char s[20];
sprintf(s, "%d", jcr.id);
fname = sdscat(fname, s);
fname = sdscat(fname, ".sparse");
FILE* fp = fopen(fname, "w");
if (!fp) {
fprintf(stderr, "Can not create sparse file");
perror("The reason is");
exit(1);
}
jcr.total_container_num = g_hash_table_size(container_utilization_monitor);
GSequence *seq = g_sequence_new(NULL);
int64_t total_size = 0;
int64_t sparse_size = 0;
/* collect sparse containers */
GHashTableIter iter;
gpointer key, value;
g_hash_table_iter_init(&iter, container_utilization_monitor);
while (g_hash_table_iter_next(&iter, &key, &value)) {
struct containerRecord* cr = (struct containerRecord*) value;
total_size += cr->size;
if((1.0*cr->size/(CONTAINER_SIZE - CONTAINER_META_SIZE))
< destor.rewrite_har_utilization_threshold){
/* It is sparse */
if (inherited_sparse_containers
&& g_hash_table_lookup(inherited_sparse_containers, &cr->cid))
/* It is an inherited sparse container */
jcr.inherited_sparse_num++;
jcr.sparse_container_num++;
sparse_size += cr->size;
g_sequence_insert_sorted(seq, cr, g_record_cmp, NULL);
}
}
/*
* If the sparse size is too large,
* we need to trim the sequence to control the rewrite ratio.
* We use sparse_size/total_size to estimate the rewrite ratio of next backup.
* However, the estimation is inaccurate (generally over-estimating), since:
* 1. the sparse size is not an accurate indicator of utilization for next backup.
* 2. self-references.
*/
while(sparse_size*1.0/total_size > destor.rewrite_har_rewrite_limit){
/*
* The expected rewrite ratio exceeds the limit.
* We trim the last several records in the sequence.
* */
GSequenceIter* iter = g_sequence_iter_prev(g_sequence_get_end_iter(seq));
struct containerRecord* r = g_sequence_get(iter);
NOTICE("Trim sparse container %lld", r->cid);
sparse_size -= r->size;
g_sequence_remove(iter);
}
GSequenceIter* sparse_iter = g_sequence_get_begin_iter(seq);
while(sparse_iter != g_sequence_get_end_iter(seq)){
struct containerRecord* r = g_sequence_get(sparse_iter);
fprintf(fp, "%lld %d\n", r->cid, r->size);
sparse_iter = g_sequence_iter_next(sparse_iter);
}
fclose(fp);
NOTICE("Record %d sparse containers, and %d of them are inherited",
g_sequence_get_length(seq), jcr.inherited_sparse_num);
g_sequence_free(seq);
sdsfree(fname);
}
