Ejemplo n.º 1
0
static int smack_backend_lookup_raw(struct smack_backend *s, struct index *idx, void *state, struct dnet_cmd *cmd)
{
	int err, fd;
	char *path;

	err = smack_lookup(s->smack, idx, &path);
	if (err < 0)
		goto err_out_exit;

	fd = open(path, O_RDONLY | O_CLOEXEC);
	if (fd < 0) {
		err = -errno;
		dnet_backend_log(DNET_LOG_ERROR, "%s: SMACK: %s: lookup-open: size: %llu: %s %d.\n",
				dnet_dump_id_str(idx->id), path,
				(unsigned long long)idx->data_size,
				strerror(-err), err);
		goto err_out_free;
	}

	err = dnet_send_file_info(state, cmd, fd, 0, idx->data_size);
	if (err)
		goto err_out_close;

	dnet_backend_log(DNET_LOG_INFO, "%s: SMACK: %s: lookup: size: %llu.\n",
			dnet_dump_id(&cmd->id), path, (unsigned long long)idx->data_size);

err_out_close:
	close(fd);
err_out_free:
	free(path);
err_out_exit:
	return err;
}
Ejemplo n.º 2
0
static int notify_complete(struct dnet_net_state *state,
			struct dnet_cmd *cmd,
			void *priv)
{
	struct dnet_io_notification *io;
	char str[64];
	struct tm tm;
	struct timeval tv;
	FILE *stream = priv;

	if (is_trans_destroyed(state, cmd))
		return 0;

	if (cmd->size != sizeof(struct dnet_io_notification))
		return 0;

	gettimeofday(&tv, NULL);
	localtime_r((time_t *)&tv.tv_sec, &tm);
	strftime(str, sizeof(str), "%F %R:%S", &tm);

	fprintf(stream, "%s.%06lu : ", str, (unsigned long)tv.tv_usec);

	io = (struct dnet_io_notification *)(cmd + 1);

	dnet_convert_io_notification(io);

	fprintf(stream, "%s: client: %s, size: %llu, offset: %llu, flags: %x\n",
			dnet_dump_id_str(io->io.id), dnet_server_convert_dnet_addr(&io->addr.addr),
			(unsigned long long)io->io.size,
			(unsigned long long)io->io.offset, io->io.flags);
	fflush(stream);

	return 0;
}
Ejemplo n.º 3
0
void throw_error(int err, const uint8_t *id, const char *format, ...)
{
	va_list args;
	va_start(args, format);
	throw_error_detail(err, dnet_dump_id_str(id), format, args);
	va_end(args);
}
Ejemplo n.º 4
0
error_info create_error(int err, const uint8_t *id, const char *format, ...)
{
	va_list args;
	va_start(args, format);
	error_info error = create_info(err, dnet_dump_id_str(id), format, args);
	va_end(args);
	return error;
}
Ejemplo n.º 5
0
static int file_write_raw(struct file_backend_root *r, struct dnet_io_attr *io)
{
	/* null byte + maximum directory length (32 bits in hex) + '/' directory prefix */
	char file[DNET_ID_SIZE * 2 + 8 + 8 + 2];
	int oflags = O_RDWR | O_CREAT | O_LARGEFILE | O_CLOEXEC;
	void *data = io + 1;
	int fd;
	ssize_t err;

	file_backend_setup_file(r, file, sizeof(file), io->id);

	if (io->flags & DNET_IO_FLAGS_APPEND)
		oflags |= O_APPEND;
	else if (!io->offset)
		oflags |= O_TRUNC;
	
	fd = open(file, oflags, 0644);
	if (fd < 0) {
		err = -errno;
		dnet_backend_log(DNET_LOG_ERROR, "%s: FILE: %s: OPEN: %zd: %s.\n",
				dnet_dump_id_str(io->id), file, err, strerror(-err));
		goto err_out_exit;
	}

	err = pwrite(fd, data, io->size, io->offset);
	if (err != (ssize_t)io->size) {
		err = -errno;
		dnet_backend_log(DNET_LOG_ERROR, "%s: FILE: %s: WRITE: %zd: offset: %llu, size: %llu: %s.\n",
			dnet_dump_id_str(io->id), file, err,
			(unsigned long long)io->offset, (unsigned long long)io->size,
			strerror(-err));
		goto err_out_close;
	}

	if (!r->sync)
		fsync(fd);

	return fd;

err_out_close:
	dnet_remove_file_if_empty_raw(file);
	close(fd);
err_out_exit:
	return err;
}
Ejemplo n.º 6
0
Archivo: app.cpp Proyecto: kod3r/grape
void queue_app_context::process(const std::string &cocaine_event, const std::vector<std::string> &chunks, cocaine::framework::response_ptr response)
{
    ioremap::elliptics::exec_context context = ioremap::elliptics::exec_context::from_raw(chunks[0].c_str(), chunks[0].size());

    std::string app;
    std::string event;
    {
        char *p = strchr((char*)context.event().c_str(), '@');
        app.assign(context.event().c_str(), p - context.event().c_str());
        event.assign(p + 1);
    }

    const std::string action_id = cocaine::format("%s %d, %s", dnet_dump_id_str(context.src_id()->id), context.src_key(), m_id.c_str());

    COCAINE_LOG_INFO(m_log, "%s, event: %s, data-size: %ld",
                     action_id.c_str(),
                     event.c_str(), context.data().size()
                    );

    if (event == "ping") {
        m_queue->final(context, std::string("ok"));

    } else if (event == "push") {
Ejemplo n.º 7
0
static int dnet_bulk_add_id(struct dnet_node *n, struct dnet_bulk_array *bulk_array, struct dnet_id *id,
				struct dnet_meta_container *mc, struct dnet_check_params *params)
{
	int err = 0;
	struct dnet_bulk_state tmp;
	struct dnet_bulk_state *state = NULL;
	struct dnet_net_state *st = dnet_state_get_first(n, id);
	struct dnet_bulk_id *bulk_id;
	struct dnet_meta_update mu;

	dnet_log(n, DNET_LOG_DEBUG, "BULK: adding ID %s to array\n", dnet_dump_id(id));
	if (!st)
		return -1;

	memcpy(&tmp.addr, &st->addr, sizeof(struct dnet_addr));
	dnet_state_put(st);

	dnet_log(n, DNET_LOG_DEBUG, "BULK: Searching state in states array\n");
	state = bsearch(&tmp, bulk_array->states, bulk_array->num, sizeof(struct dnet_bulk_state), dnet_compare_bulk_state);
	if (!state)
		return -1;

	if (!dnet_get_meta_update(n, mc, &mu))
		return -ENOENT;

	dnet_log(n, DNET_LOG_DEBUG, "BULK: addr = %s state->num = %d\n", dnet_server_convert_dnet_addr(&state->addr), state->num);
	//pthread_mutex_lock(&state->state_lock);
	if (state->num >= DNET_BULK_IDS_SIZE || state->num < 0)
		goto err_out_unlock;

	bulk_id = &state->ids[state->num];
	memset(bulk_id, 0, sizeof(struct dnet_bulk_id));

	memcpy(&bulk_id->id, &id->id, DNET_ID_SIZE);

	dnet_log(n, DNET_LOG_DEBUG, "BULK: ID: %s, last_update->tsec=%llu, last_update->tnsec=%llu, flags=%02llx\n", 
			dnet_dump_id_str(bulk_id->id.id), (unsigned long long)mu.tm.tsec, (unsigned long long)mu.tm.tnsec,
			(unsigned long long)mu.flags);

	dnet_convert_meta_update(&mu);

	memcpy(&bulk_id->last_update, &mu, sizeof(struct dnet_meta_update));

	state->num++;

	dnet_log(n, DNET_LOG_DEBUG, "BULK: addr = %s state->num = %d\n", dnet_server_convert_dnet_addr(&state->addr), state->num);
	if (state->num == DNET_BULK_IDS_SIZE) {
		err = dnet_request_bulk_check(n, state, params);
		state->num = 0;
		if (err)
			goto err_out_unlock;
	}

	//pthread_mutex_unlock(&state->state_lock);

	return 0;

err_out_unlock:
	//pthread_mutex_unlock(&state->state_lock);
	return -2;
}
Ejemplo n.º 8
0
static int dnet_bulk_check_complete_single(struct dnet_net_state *state, struct dnet_bulk_id *ids,
						int remote_group, struct dnet_bulk_check_priv *p)
{
	struct dnet_session *s = dnet_session_create(state->n);
	struct dnet_meta_container mc;
	struct dnet_meta_container temp_mc;
	struct dnet_meta_update *mu;
	struct dnet_meta *mg;
	struct dnet_id id;
	char *tmpdata = NULL;
	int *groups, group_num = 1;
	int err = -EINVAL, error = 0;
	int i = 0;
	int my_group, lastest_group = -1;
	struct dnet_meta_update lastest_mu, my_mu;
	struct timeval current_ts;
	int removed_in_all = 1, updated = 0;
	int lastest = 0;
	uint64_t cflags = 0;
	uint64_t ioflags = 0;


	my_group = state->n->id.group_id;

	dnet_log(state->n, DNET_LOG_DEBUG, "BULK: checking ID %s\n", dnet_dump_id_str(ids->id.id));
	err = -ENOENT;
	error = 0;

	dnet_setup_id(&mc.id, my_group, ids->id.id);

	err = state->n->cb->meta_read(state->n->cb->command_private, &ids->id, &mc.data);
	if (err <= 0) {
		if (err == 0)
			err = -ENOENT;
		goto err_out_continue;
	}
	mc.size = err;

	/* Set current group meta_update as lastest_mu */
	if (!dnet_get_meta_update(state->n, &mc, &my_mu)) {
		dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: meta_update structure doesn't exist for group %d\n",
				dnet_dump_id_str(ids->id.id), my_group);
		err = -ENOENT;
		goto err_out_kcfree;
	}
	dnet_convert_meta_update(&my_mu);
	memcpy(&lastest_mu, &my_mu, sizeof(struct dnet_meta_update));
	lastest_group = my_group;

	/* Get group list */
	if (p->group_num) {
		/* groups came from dnet_check utility */
		groups = p->groups;
		group_num = p->group_num;
	} else {
		mg = dnet_meta_search(state->n, &mc, DNET_META_GROUPS);
		if (!mg) {
			dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: DNET_META_GROUPS structure doesn't exist\n", dnet_dump_id_str(ids->id.id));
			err = -ENOENT;
			goto err_out_kcfree;
		}
		dnet_convert_meta(mg);
		if (mg->size % sizeof(int)) {
			dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: DNET_META_GROUPS structure is corrupted\n", dnet_dump_id_str(ids->id.id));
			err = -1;
			goto err_out_kcfree;
		}
		group_num = mg->size / sizeof(int);
		groups = (int *)mg->data;
		dnet_convert_meta(mg);
	}

	/* Read temporary meta */
	temp_mc.data = malloc(sizeof(struct dnet_meta_update) * group_num);
	if (!temp_mc.data) {
		err = -ENOMEM;
		dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: could not allocate memory for temp UPDATE_META\n", dnet_dump_id_str(ids->id.id));
		goto err_out_kcfree;
	}
	memset(temp_mc.data, 0, sizeof(struct dnet_meta_update) * group_num);
	temp_mc.size = sizeof(struct dnet_meta_update) * group_num;

	err = dnet_db_read_raw(p->db->b, &ids->id, (void **)&tmpdata);
	if (err <= 0) {
		if (err < 0 && err != -2)
			goto err_out_free;
		/* No data in temp meta was stored. Placing local meta_update at the beginning */
		mu = temp_mc.data;
		mu[0].group_id = my_group;
		mu[0].tm = my_mu.tm;
		mu[0].flags = my_mu.flags;
		
	} else {
		if (err > (int)(sizeof(struct dnet_meta_update) * group_num)) {
			dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: too many data stored in temp meta\n",  dnet_dump_id_str(ids->id.id));
			err = -ENOMEM;
			goto err_out_free;
		}
		memcpy(temp_mc.data, tmpdata, err);
	}

	/* Update temp meta with received group */
	mu = temp_mc.data;
	updated = 0;
	lastest = 0;
	for (i = 0; i < group_num; ++i) {
		if (mu[i].group_id == remote_group) {
			mu[i].tm = ids->last_update.tm;
			mu[i].flags = ids->last_update.flags;
			updated = 1;
		}

		if (mu[i].group_id == 0)
			break;

		if (!(mu[i].flags & DNET_IO_FLAGS_REMOVED))
			removed_in_all = 0;

		if (((mu[i].tm.tsec > mu[lastest].tm.tsec)
				|| ((mu[i].tm.tsec == mu[lastest].tm.tsec) && (mu[i].tm.tnsec > mu[lastest].tm.tnsec)))
			&& i != lastest) {
			lastest = i;
			lastest_group = groups[i];
		}
	
	}
	if (!updated && i == group_num) {
		dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: no space left to save group in temp meta!\n", dnet_dump_id_str(ids->id.id));
		err = -ENOMEM;
		goto err_out_free;
	}
	if (!updated) {
		mu[i].group_id = remote_group;
		mu[i].tm = ids->last_update.tm;
		mu[i].flags = ids->last_update.flags;

		if (((mu[i].tm.tsec > mu[lastest].tm.tsec)
				|| ((mu[i].tm.tsec == mu[lastest].tm.tsec) && (mu[i].tm.tnsec > mu[lastest].tm.tnsec)))
			&& i != lastest) {
			lastest = i;
			lastest_group = groups[i];
		}

		++i;
	}

	/* Not all groups processed yet */
	if (i < group_num) {
		err = 0;
		err = dnet_db_write_raw(p->db->b, &ids->id, temp_mc.data, temp_mc.size);
		if (err) {
			dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: unable to save temp meta, err: %d\n", dnet_dump_id_str(ids->id.id), err);
		}
		goto err_out_free;
	}

	/* Check if removal_delay second has gone since object was marked as REMOVED */
	if (removed_in_all) {
		gettimeofday(&current_ts, NULL);
		if (((uint64_t)current_ts.tv_sec < mu[lastest].tm.tsec) 
			|| ((uint64_t)current_ts.tv_sec - mu[lastest].tm.tsec) < (uint64_t)(state->n->removal_delay * 3600 * 24))
			removed_in_all = 0;
	}

	/* TODO: receive newer files from remote groups
	 *
	 * Yep, we should read it locally and send it to other groups too
	 */
	if ((lastest_group != my_group) && !(mu[lastest].flags & DNET_IO_FLAGS_REMOVED)) {
		dnet_log(state->n, DNET_LOG_DEBUG, "BULK: %s: File on remote group %d is newer, skipping this file\n",
				dnet_dump_id_str(ids->id.id), lastest_group);
		err = 0;
		goto err_out_free;
	}

	for (i = 0; i < group_num; ++i) {
		err = 0;
		if (mu[i].group_id == my_group)
			continue;

		dnet_session_set_groups(s, (int *)&mu[i].group_id, 1);
		dnet_setup_id(&id, mu[i].group_id, ids->id.id);
		id.type = -1;

		if (mu[lastest].flags & DNET_IO_FLAGS_REMOVED) {
			if (removed_in_all) {
				dnet_log(state->n, DNET_LOG_DEBUG, "BULK: dnet_remove_object_now %s in group %d, err=%d\n",
						dnet_dump_id(&id), mu[i].group_id, err);
				err = dnet_remove_object_now(s, &id, cflags, ioflags);
			} else {
				if (!(mu[i].flags & DNET_IO_FLAGS_REMOVED)) {
					err = dnet_remove_object(s, &id, NULL, NULL, cflags, ioflags);
					dnet_log(state->n, DNET_LOG_DEBUG, "BULK: dnet_remove_object %s in group %d err=%d\n",
							dnet_dump_id(&id), mu[i].group_id, err);
				}
			}
			if (err < 0)
				goto err_out_cont2;
		} else {
			if ((mu[i].tm.tsec < mu[lastest].tm.tsec) || ((mu[i].tm.tsec == mu[lastest].tm.tsec) &&
									((mu[i].tm.tnsec < mu[lastest].tm.tnsec)))) {
				err = state->n->cb->send(state, state->n->cb->command_private, &id);

				if (err)
					goto err_out_cont2;

				err = dnet_meta_update_check_status_raw(state->n, &mc);
				if (err)
					goto err_out_cont2;

				memcpy(&mc.id, &id, sizeof(struct dnet_id));
				err = dnet_write_metadata(s, &mc, 1, cflags);
				dnet_log(state->n, DNET_LOG_DEBUG, "BULK: dnet_write_metadata %s in group %d, err=%d\n",
						dnet_dump_id(&id), my_group, err);

				if (err < 0)
					goto err_out_cont2;
			}
		}
err_out_cont2:
		if (err < 0)
			dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: Error during sending transaction to group %d, err=%d\n",
					dnet_dump_id_str(ids->id.id), groups[i], err);
		if (!error && err < 0)
			error = err;
	}

	if (mu[lastest].flags & DNET_IO_FLAGS_REMOVED) {
		if (removed_in_all) {
			err = dnet_merge_remove_local(state->n, &mc.id, 0);
		} else if (!(my_mu.flags & DNET_IO_FLAGS_REMOVED)) {
			err = dnet_merge_remove_local(state->n, &mc.id, 1);
		}
	}

	if (!(mu[lastest].flags & DNET_IO_FLAGS_REMOVED) && !error) {
		err = dnet_meta_update_check_status(state->n, &mc);
		if (err) {
			dnet_log(state->n, DNET_LOG_ERROR, "BULK: %s: couldn't update meta CHECK_STATUS\n", dnet_dump_id_str(ids->id.id));
		}
	}

	if (group_num > 2) {
		err = dnet_db_remove_raw(p->db->b, &ids->id, 1);
		if (!err) {
			dnet_log_raw(state->n, DNET_LOG_ERROR, "BULK: %s: DB: failed to remove temp_meta object, err: %d.\n",
				dnet_dump_id(&mc.id), err);
		}
	}

	if (error > 0)
		error = 0;
err_out_free:
	free(temp_mc.data);
err_out_kcfree:
	free(mc.data);
	if (tmpdata)
		free(tmpdata);
err_out_continue:
	if (error < 0) {
		dnet_log(state->n, DNET_LOG_ERROR, "Failed to check ID %s to %s, err=%d\n", dnet_dump_id_str(ids->id.id),
				dnet_state_dump_addr(state), error);
	}
	if (i == group_num) {
		dnet_counter_inc(state->n, DNET_CNTR_NODE_CHECK_COPY, error);
	}

	return error;
}
Ejemplo n.º 9
0
int dnet_cmd_bulk_check(struct dnet_net_state *orig, struct dnet_cmd *cmd, void *data)
{
	struct dnet_bulk_id *ids = (struct dnet_bulk_id *)data;
	struct dnet_meta_container mc;
	struct dnet_meta_update mu;
	struct dnet_id raw;
	int i;
	int err = 0;
	int num;

	if (!(cmd->size % sizeof(struct dnet_bulk_id))) {
		num = cmd->size / sizeof(struct dnet_bulk_id);

		dnet_log(orig->n, DNET_LOG_DEBUG, "BULK: received %d entries\n", num);

		for (i = 0; i < num; ++i) {

			/* Send empty reply every DNET_BULK_CHECK_PING records to prevent timeout */
			if (i % DNET_BULK_CHECK_PING == 0 && i > 0) {
				dnet_send_reply(orig, cmd, NULL, 0, 1);
			}

			dnet_log(orig->n, DNET_LOG_DEBUG, "BULK: processing ID %s\n", dnet_dump_id_str(ids[i].id.id));
			mc.data = NULL;
			dnet_setup_id(&mc.id, 0, ids[i].id.id);
			err = orig->n->cb->meta_read(orig->n->cb->command_private, &ids[i].id, &mc.data);
			if (mc.data) {
				mc.size = err;
				dnet_log(orig->n, DNET_LOG_DEBUG, "BULK: %d bytes of metadata found, searching for META_UPDATE group_id=%d\n",
						mc.size, orig->n->st->idc->group->group_id);
				if (dnet_get_meta_update(orig->n, &mc, &mu))
				{
					dnet_convert_meta_update(&ids[i].last_update);
					dnet_log(orig->n, DNET_LOG_DEBUG, "BULK: mu.tsec=%lu, mu.tnsec=%lu, mu.flags=%02lx\n",
							(unsigned long)mu.tm.tsec, (unsigned long)mu.tm.tnsec, (unsigned long)mu.flags);
					dnet_log(orig->n, DNET_LOG_DEBUG,
							"BULK: last_update.tsec=%lu, last_update.tnsec=%lu, last_update.flags=%02lx\n",
							(unsigned long)ids[i].last_update.tm.tsec, (unsigned long)ids[i].last_update.tm.tnsec,
							(unsigned long)ids[i].last_update.flags);

					if ((mu.flags & DNET_IO_FLAGS_REMOVED) || (mu.tm.tsec < ids[i].last_update.tm.tsec) || 
						((mu.tm.tnsec < ids[i].last_update.tm.tnsec) && (mu.tm.tsec == ids[i].last_update.tm.tsec))) {
						err = 0;
					} else {
						/* File is not needed to be updated */
						dnet_setup_id(&raw, orig->n->id.group_id, ids[i].id.id);
						err = dnet_stat_local(orig, &raw);
						if (err) {
							/* File was not found in the storage */
							mu.tm.tsec = 1;
							mu.flags = 0;
						} else {
							err = dnet_meta_update_check_status(orig->n, &mc);
							if (err) {
								dnet_log(orig->n, DNET_LOG_ERROR,
										"BULK: %s: couldn't update meta CHECK_STATUS err: %d\n",
										dnet_dump_id_str(ids[i].id.id), err);
							}
						}
					}

					memcpy(&ids[i].last_update, &mu, sizeof(struct dnet_meta_update));
					dnet_convert_meta_update(&ids[i].last_update);
				}
				free(mc.data);
			} else {
				/* Meta is not present - set timestamp to very old one */
				dnet_convert_meta_update(&ids[i].last_update);
				ids[i].last_update.tm.tsec = 1;
				ids[i].last_update.flags = 0;
				dnet_convert_meta_update(&ids[i].last_update);
			}
		}
	} else {
		dnet_log(orig->n, DNET_LOG_ERROR, "BULK: received corrupted data, size = %llu, sizeof(dnet_bulk_id) = %zu\n",
				(unsigned long long)cmd->size, sizeof(struct dnet_bulk_id));
		err = -1;
		goto err_out_exit;
	}

	return dnet_send_reply(orig, cmd, data, sizeof(struct dnet_bulk_id) * num, 0);

err_out_exit:
	return err;
}