Пример #1
0
struct dnet_meta *dnet_meta_search(struct dnet_node *n, struct dnet_meta_container *mc, uint32_t type)
{
    void *data = mc->data;
    uint32_t size = mc->size;
    struct dnet_meta m, *found = NULL;

    while (size) {
        if (size < sizeof(struct dnet_meta)) {
            dnet_map_log(n, DNET_LOG_ERROR, "Metadata size %u is too small, min %zu, searching for type 0x%x.\n",
                         size, sizeof(struct dnet_meta), type);
            break;
        }

        m = *(struct dnet_meta *)data;
        dnet_convert_meta(&m);

        if (m.size + sizeof(struct dnet_meta) > size) {
            dnet_map_log(n, DNET_LOG_ERROR, "Metadata entry broken: entry size %u, type: 0x%x, struct size: %zu, "
                         "total size left: %u, searching for type: 0x%x.\n",
                         m.size, m.type, sizeof(struct dnet_meta), size, type);
            break;
        }

        if (m.type == type) {
            found = data;
            break;
        }

        data += m.size + sizeof(struct dnet_meta);
        size -= m.size + sizeof(struct dnet_meta);
    }

    return found;
}
Пример #2
0
struct dnet_meta_update *dnet_get_meta_update(struct dnet_node *n, struct dnet_meta_container *mc,
        struct dnet_meta_update *meta_update)
{
    struct dnet_meta m;
    void *data = mc->data;
    uint32_t size = mc->size;
    struct dnet_meta_update *mu;
    int num;

    while (size) {
        if (size < sizeof(struct dnet_meta)) {
            dnet_log(n, DNET_LOG_ERROR, "Metadata size %u is too small, min %zu, searching for type 0x%x.\n",
                     size, sizeof(struct dnet_meta), DNET_META_UPDATE);
            return NULL;
        }

        m = *(struct dnet_meta *)data;
        dnet_convert_meta(&m);

        if (m.size + sizeof(struct dnet_meta) > size) {
            dnet_log(n, DNET_LOG_ERROR, "Metadata entry broken: entry size %u, type: 0x%x, struct size: %zu, "
                     "total size left: %u, searching for type: 0x%x.\n",
                     m.size, m.type, sizeof(struct dnet_meta), size, DNET_META_UPDATE);
            return NULL;
        }

        if (m.type == DNET_META_UPDATE) {
            mu = (struct dnet_meta_update *)(data + sizeof(struct dnet_meta));

            num = m.size / sizeof(struct dnet_meta_update);
            if (num >= 0) {
                if (meta_update) {
                    memcpy(meta_update, &mu[0], sizeof(struct dnet_meta_update));
                    dnet_convert_meta_update(meta_update);
                }
                return &mu[0];
            }
        }

        data += m.size + sizeof(struct dnet_meta);
        size -= m.size + sizeof(struct dnet_meta);
    }
    return NULL;
}
Пример #3
0
void dnet_create_meta_update(struct dnet_meta *m, struct timespec *ts, uint64_t flags_set, uint64_t flags_clear)
{
    struct dnet_meta_update *mu = (struct dnet_meta_update *)m->data;

    m->size = sizeof(struct dnet_meta_update);
    m->type = DNET_META_UPDATE;

    if (!ts) {
        dnet_current_time(&mu->tm);
    } else {
        mu->tm.tsec = ts->tv_sec;
        mu->tm.tnsec = ts->tv_nsec;
    }

    mu->flags = 0;
    mu->flags |= flags_set;
    mu->flags &= ~flags_clear;

    dnet_convert_meta_update(mu);
    dnet_convert_meta(m);
}
Пример #4
0
int dnet_update_ts_metadata_raw(struct dnet_meta_container *mc, uint64_t flags_set, uint64_t flags_clear)
{
    struct dnet_meta m;
    void *data = mc->data;
    uint32_t size = mc->size;
    struct dnet_meta_update *mu;

    while (size) {
        if (size < sizeof(struct dnet_meta)) {
            return -1;
        }

        m = *(struct dnet_meta *)data;
        dnet_convert_meta(&m);

        if (m.size + sizeof(struct dnet_meta) > size) {
            return -1;
        }

        if (m.type == DNET_META_UPDATE) {
            mu = (struct dnet_meta_update *)(data + sizeof(struct dnet_meta));

            dnet_convert_meta_update(mu);

            dnet_current_time(&mu->tm);
            mu->flags |= flags_set;
            mu->flags &= ~flags_clear;

            dnet_convert_meta_update(mu);
        }

        data += m.size + sizeof(struct dnet_meta);
        size -= m.size + sizeof(struct dnet_meta);
    }

    return -ENOENT;
}
Пример #5
0
static int dnet_bulk_check_complete_single(struct dnet_net_state *state, struct dnet_bulk_id *ids,
						int remote_group, struct dnet_bulk_check_priv *p)
{
	struct dnet_session *s = dnet_session_create(state->n);
	struct dnet_meta_container mc;
	struct dnet_meta_container temp_mc;
	struct dnet_meta_update *mu;
	struct dnet_meta *mg;
	struct dnet_id id;
	char *tmpdata = NULL;
	int *groups, group_num = 1;
	int err = -EINVAL, error = 0;
	int i = 0;
	int my_group, lastest_group = -1;
	struct dnet_meta_update lastest_mu, my_mu;
	struct timeval current_ts;
	int removed_in_all = 1, updated = 0;
	int lastest = 0;
	uint64_t cflags = 0;
	uint64_t ioflags = 0;


	my_group = state->n->id.group_id;

	dnet_log(state->n, DNET_LOG_DEBUG, "BULK: checking ID %s\n", dnet_dump_id_str(ids->id.id));
	err = -ENOENT;
	error = 0;

	dnet_setup_id(&mc.id, my_group, ids->id.id);

	err = state->n->cb->meta_read(state->n->cb->command_private, &ids->id, &mc.data);
	if (err <= 0) {
		if (err == 0)
			err = -ENOENT;
		goto err_out_continue;
	}
	mc.size = err;

	/* Set current group meta_update as lastest_mu */
	if (!dnet_get_meta_update(state->n, &mc, &my_mu)) {
		dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: meta_update structure doesn't exist for group %d\n",
				dnet_dump_id_str(ids->id.id), my_group);
		err = -ENOENT;
		goto err_out_kcfree;
	}
	dnet_convert_meta_update(&my_mu);
	memcpy(&lastest_mu, &my_mu, sizeof(struct dnet_meta_update));
	lastest_group = my_group;

	/* Get group list */
	if (p->group_num) {
		/* groups came from dnet_check utility */
		groups = p->groups;
		group_num = p->group_num;
	} else {
		mg = dnet_meta_search(state->n, &mc, DNET_META_GROUPS);
		if (!mg) {
			dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: DNET_META_GROUPS structure doesn't exist\n", dnet_dump_id_str(ids->id.id));
			err = -ENOENT;
			goto err_out_kcfree;
		}
		dnet_convert_meta(mg);
		if (mg->size % sizeof(int)) {
			dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: DNET_META_GROUPS structure is corrupted\n", dnet_dump_id_str(ids->id.id));
			err = -1;
			goto err_out_kcfree;
		}
		group_num = mg->size / sizeof(int);
		groups = (int *)mg->data;
		dnet_convert_meta(mg);
	}

	/* Read temporary meta */
	temp_mc.data = malloc(sizeof(struct dnet_meta_update) * group_num);
	if (!temp_mc.data) {
		err = -ENOMEM;
		dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: could not allocate memory for temp UPDATE_META\n", dnet_dump_id_str(ids->id.id));
		goto err_out_kcfree;
	}
	memset(temp_mc.data, 0, sizeof(struct dnet_meta_update) * group_num);
	temp_mc.size = sizeof(struct dnet_meta_update) * group_num;

	err = dnet_db_read_raw(p->db->b, &ids->id, (void **)&tmpdata);
	if (err <= 0) {
		if (err < 0 && err != -2)
			goto err_out_free;
		/* No data in temp meta was stored. Placing local meta_update at the beginning */
		mu = temp_mc.data;
		mu[0].group_id = my_group;
		mu[0].tm = my_mu.tm;
		mu[0].flags = my_mu.flags;
		
	} else {
		if (err > (int)(sizeof(struct dnet_meta_update) * group_num)) {
			dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: too many data stored in temp meta\n",  dnet_dump_id_str(ids->id.id));
			err = -ENOMEM;
			goto err_out_free;
		}
		memcpy(temp_mc.data, tmpdata, err);
	}

	/* Update temp meta with received group */
	mu = temp_mc.data;
	updated = 0;
	lastest = 0;
	for (i = 0; i < group_num; ++i) {
		if (mu[i].group_id == remote_group) {
			mu[i].tm = ids->last_update.tm;
			mu[i].flags = ids->last_update.flags;
			updated = 1;
		}

		if (mu[i].group_id == 0)
			break;

		if (!(mu[i].flags & DNET_IO_FLAGS_REMOVED))
			removed_in_all = 0;

		if (((mu[i].tm.tsec > mu[lastest].tm.tsec)
				|| ((mu[i].tm.tsec == mu[lastest].tm.tsec) && (mu[i].tm.tnsec > mu[lastest].tm.tnsec)))
			&& i != lastest) {
			lastest = i;
			lastest_group = groups[i];
		}
	
	}
	if (!updated && i == group_num) {
		dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: no space left to save group in temp meta!\n", dnet_dump_id_str(ids->id.id));
		err = -ENOMEM;
		goto err_out_free;
	}
	if (!updated) {
		mu[i].group_id = remote_group;
		mu[i].tm = ids->last_update.tm;
		mu[i].flags = ids->last_update.flags;

		if (((mu[i].tm.tsec > mu[lastest].tm.tsec)
				|| ((mu[i].tm.tsec == mu[lastest].tm.tsec) && (mu[i].tm.tnsec > mu[lastest].tm.tnsec)))
			&& i != lastest) {
			lastest = i;
			lastest_group = groups[i];
		}

		++i;
	}

	/* Not all groups processed yet */
	if (i < group_num) {
		err = 0;
		err = dnet_db_write_raw(p->db->b, &ids->id, temp_mc.data, temp_mc.size);
		if (err) {
			dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: unable to save temp meta, err: %d\n", dnet_dump_id_str(ids->id.id), err);
		}
		goto err_out_free;
	}

	/* Check if removal_delay second has gone since object was marked as REMOVED */
	if (removed_in_all) {
		gettimeofday(&current_ts, NULL);
		if (((uint64_t)current_ts.tv_sec < mu[lastest].tm.tsec) 
			|| ((uint64_t)current_ts.tv_sec - mu[lastest].tm.tsec) < (uint64_t)(state->n->removal_delay * 3600 * 24))
			removed_in_all = 0;
	}

	/* TODO: receive newer files from remote groups
	 *
	 * Yep, we should read it locally and send it to other groups too
	 */
	if ((lastest_group != my_group) && !(mu[lastest].flags & DNET_IO_FLAGS_REMOVED)) {
		dnet_log(state->n, DNET_LOG_DEBUG, "BULK: %s: File on remote group %d is newer, skipping this file\n",
				dnet_dump_id_str(ids->id.id), lastest_group);
		err = 0;
		goto err_out_free;
	}

	for (i = 0; i < group_num; ++i) {
		err = 0;
		if (mu[i].group_id == my_group)
			continue;

		dnet_session_set_groups(s, (int *)&mu[i].group_id, 1);
		dnet_setup_id(&id, mu[i].group_id, ids->id.id);
		id.type = -1;

		if (mu[lastest].flags & DNET_IO_FLAGS_REMOVED) {
			if (removed_in_all) {
				dnet_log(state->n, DNET_LOG_DEBUG, "BULK: dnet_remove_object_now %s in group %d, err=%d\n",
						dnet_dump_id(&id), mu[i].group_id, err);
				err = dnet_remove_object_now(s, &id, cflags, ioflags);
			} else {
				if (!(mu[i].flags & DNET_IO_FLAGS_REMOVED)) {
					err = dnet_remove_object(s, &id, NULL, NULL, cflags, ioflags);
					dnet_log(state->n, DNET_LOG_DEBUG, "BULK: dnet_remove_object %s in group %d err=%d\n",
							dnet_dump_id(&id), mu[i].group_id, err);
				}
			}
			if (err < 0)
				goto err_out_cont2;
		} else {
			if ((mu[i].tm.tsec < mu[lastest].tm.tsec) || ((mu[i].tm.tsec == mu[lastest].tm.tsec) &&
									((mu[i].tm.tnsec < mu[lastest].tm.tnsec)))) {
				err = state->n->cb->send(state, state->n->cb->command_private, &id);

				if (err)
					goto err_out_cont2;

				err = dnet_meta_update_check_status_raw(state->n, &mc);
				if (err)
					goto err_out_cont2;

				memcpy(&mc.id, &id, sizeof(struct dnet_id));
				err = dnet_write_metadata(s, &mc, 1, cflags);
				dnet_log(state->n, DNET_LOG_DEBUG, "BULK: dnet_write_metadata %s in group %d, err=%d\n",
						dnet_dump_id(&id), my_group, err);

				if (err < 0)
					goto err_out_cont2;
			}
		}
err_out_cont2:
		if (err < 0)
			dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: Error during sending transaction to group %d, err=%d\n",
					dnet_dump_id_str(ids->id.id), groups[i], err);
		if (!error && err < 0)
			error = err;
	}

	if (mu[lastest].flags & DNET_IO_FLAGS_REMOVED) {
		if (removed_in_all) {
			err = dnet_merge_remove_local(state->n, &mc.id, 0);
		} else if (!(my_mu.flags & DNET_IO_FLAGS_REMOVED)) {
			err = dnet_merge_remove_local(state->n, &mc.id, 1);
		}
	}

	if (!(mu[lastest].flags & DNET_IO_FLAGS_REMOVED) && !error) {
		err = dnet_meta_update_check_status(state->n, &mc);
		if (err) {
			dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: couldn't update meta CHECK_STATUS\n", dnet_dump_id_str(ids->id.id));
		}
	}

	if (group_num > 2) {
		err = dnet_db_remove_raw(p->db->b, &ids->id, 1);
		if (!err) {
			dnet_log_raw(state->n, DNET_LOG_ERROR, "BULK: %s: DB: failed to remove temp_meta object, err: %d.\n",
				dnet_dump_id(&mc.id), err);
		}
	}

	if (error > 0)
		error = 0;
err_out_free:
	free(temp_mc.data);
err_out_kcfree:
	free(mc.data);
	if (tmpdata)
		free(tmpdata);
err_out_continue:
	if (error < 0) {
		dnet_log(state->n, DNET_LOG_ERROR, "Failed to check ID %s to %s, err=%d\n", dnet_dump_id_str(ids->id.id),
				dnet_state_dump_addr(state), error);
	}
	if (i == group_num) {
		dnet_counter_inc(state->n, DNET_CNTR_NODE_CHECK_COPY, error);
	}

	return error;
}