예제 #1
0
/*
 * Copy a file from source to target, between 'begin' and 'end' offsets.
 */
static void
copy_file_range(const char *path, off_t begin, off_t end, bool trunc)
{
	char		buf[BLCKSZ];
	char		srcpath[MAXPGPATH];
	int			srcfd;

	snprintf(srcpath, sizeof(srcpath), "%s/%s", datadir_source, path);

	srcfd = open(srcpath, O_RDONLY | PG_BINARY, 0);
	if (srcfd < 0)
	{
		fprintf(stderr, "could not open source file \"%s\": %s\n", srcpath, strerror(errno));
		exit(1);
	}

	if (lseek(srcfd, begin, SEEK_SET) == -1)
	{
		fprintf(stderr, "could not seek in source file: %s\n", strerror(errno));
		exit(1);
	}

	open_target_file(path, trunc);

	while (end - begin > 0)
	{
		int		readlen;
		int		len;

		if (end - begin > sizeof(buf))
			len = sizeof(buf);
		else
			len = end - begin;

		readlen = read(srcfd, buf, len);

		if (readlen < 0)
		{
			fprintf(stderr, "could not read file \"%s\": %s\n", srcpath, strerror(errno));
			exit(1);
		}
		else if (readlen == 0)
		{
			fprintf(stderr, "unexpected EOF while reading file \"%s\"\n", srcpath);
			exit(1);
		}

		write_file_range(buf, begin, readlen);
		begin += readlen;
	}
}
예제 #2
0
/*
 * Copy a file from source to target, between 'begin' and 'end' offsets.
 *
 * If 'trunc' is true, any existing file with the same name is truncated.
 */
static void
rewind_copy_file_range(const char *path, off_t begin, off_t end, bool trunc)
{
	PGAlignedBlock buf;
	char		srcpath[MAXPGPATH];
	int			srcfd;

	snprintf(srcpath, sizeof(srcpath), "%s/%s", datadir_source, path);

	srcfd = open(srcpath, O_RDONLY | PG_BINARY, 0);
	if (srcfd < 0)
		pg_fatal("could not open source file \"%s\": %m",
				 srcpath);

	if (lseek(srcfd, begin, SEEK_SET) == -1)
		pg_fatal("could not seek in source file: %m");

	open_target_file(path, trunc);

	while (end - begin > 0)
	{
		int			readlen;
		int			len;

		if (end - begin > sizeof(buf))
			len = sizeof(buf);
		else
			len = end - begin;

		readlen = read(srcfd, buf.data, len);

		if (readlen < 0)
			pg_fatal("could not read file \"%s\": %m",
					 srcpath);
		else if (readlen == 0)
			pg_fatal("unexpected EOF while reading file \"%s\"", srcpath);

		write_target_range(buf.data, begin, readlen);
		begin += readlen;
	}

	if (close(srcfd) != 0)
		pg_fatal("could not close file \"%s\": %m", srcpath);
}
예제 #3
0
/*
 * Fetch all changed blocks from remote source data directory.
 */
void
libpq_executeFileMap(filemap_t *map)
{
	file_entry_t *entry;
	const char *sql;
	PGresult   *res;
	int			i;

	/*
	 * First create a temporary table, and load it with the blocks that we
	 * need to fetch.
	 */
	sql = "CREATE TEMPORARY TABLE fetchchunks(path text, begin int8, len int4);";
	res = PQexec(conn, sql);

	if (PQresultStatus(res) != PGRES_COMMAND_OK)
		pg_fatal("could not create temporary table: %s",
				 PQresultErrorMessage(res));
	PQclear(res);

	sql = "COPY fetchchunks FROM STDIN";
	res = PQexec(conn, sql);

	if (PQresultStatus(res) != PGRES_COPY_IN)
		pg_fatal("could not send file list: %s",
				 PQresultErrorMessage(res));
	PQclear(res);

	for (i = 0; i < map->narray; i++)
	{
		entry = map->array[i];

		/* If this is a relation file, copy the modified blocks */
		execute_pagemap(&entry->pagemap, entry->path);

		switch (entry->action)
		{
			case FILE_ACTION_NONE:
				/* nothing else to do */
				break;

			case FILE_ACTION_COPY:
				/* Truncate the old file out of the way, if any */
				open_target_file(entry->path, true);
				fetch_file_range(entry->path, 0, entry->newsize);
				break;

			case FILE_ACTION_TRUNCATE:
				truncate_target_file(entry->path, entry->newsize);
				break;

			case FILE_ACTION_COPY_TAIL:
				fetch_file_range(entry->path, entry->oldsize, entry->newsize);
				break;

			case FILE_ACTION_REMOVE:
				remove_target(entry);
				break;

			case FILE_ACTION_CREATE:
				create_target(entry);
				break;
		}
	}

	if (PQputCopyEnd(conn, NULL) != 1)
		pg_fatal("could not send end-of-COPY: %s",
				 PQerrorMessage(conn));

	while ((res = PQgetResult(conn)) != NULL)
	{
		if (PQresultStatus(res) != PGRES_COMMAND_OK)
			pg_fatal("unexpected result while sending file list: %s",
					 PQresultErrorMessage(res));
		PQclear(res);
	}

	/*
	 * We've now copied the list of file ranges that we need to fetch to the
	 * temporary table. Now, actually fetch all of those ranges.
	 */
	sql =
		"SELECT path, begin, \n"
		"  pg_read_binary_file(path, begin, len, true) AS chunk\n"
		"FROM fetchchunks\n";

	receiveFileChunks(sql);
}
예제 #4
0
/*----
 * Runs a query, which returns pieces of files from the remote source data
 * directory, and overwrites the corresponding parts of target files with
 * the received parts. The result set is expected to be of format:
 *
 * path		text	-- path in the data directory, e.g "base/1/123"
 * begin	int8	-- offset within the file
 * chunk	bytea	-- file content
 *----
 */
static void
receiveFileChunks(const char *sql)
{
	PGresult   *res;

	if (PQsendQueryParams(conn, sql, 0, NULL, NULL, NULL, NULL, 1) != 1)
		pg_fatal("could not send query: %s", PQerrorMessage(conn));

	pg_log(PG_DEBUG, "getting file chunks\n");

	if (PQsetSingleRowMode(conn) != 1)
		pg_fatal("could not set libpq connection to single row mode\n");

	while ((res = PQgetResult(conn)) != NULL)
	{
		char	   *filename;
		int			filenamelen;
		int64		chunkoff;
		char		chunkoff_str[32];
		int			chunksize;
		char	   *chunk;

		switch (PQresultStatus(res))
		{
			case PGRES_SINGLE_TUPLE:
				break;

			case PGRES_TUPLES_OK:
				PQclear(res);
				continue;		/* final zero-row result */

			default:
				pg_fatal("unexpected result while fetching remote files: %s",
						 PQresultErrorMessage(res));
		}

		/* sanity check the result set */
		if (PQnfields(res) != 3 || PQntuples(res) != 1)
			pg_fatal("unexpected result set size while fetching remote files\n");

		if (PQftype(res, 0) != TEXTOID ||
			PQftype(res, 1) != INT8OID ||
			PQftype(res, 2) != BYTEAOID)
		{
			pg_fatal("unexpected data types in result set while fetching remote files: %u %u %u\n",
					 PQftype(res, 0), PQftype(res, 1), PQftype(res, 2));
		}

		if (PQfformat(res, 0) != 1 &&
			PQfformat(res, 1) != 1 &&
			PQfformat(res, 2) != 1)
		{
			pg_fatal("unexpected result format while fetching remote files\n");
		}

		if (PQgetisnull(res, 0, 0) ||
			PQgetisnull(res, 0, 1))
		{
			pg_fatal("unexpected null values in result while fetching remote files\n");
		}

		if (PQgetlength(res, 0, 1) != sizeof(int64))
			pg_fatal("unexpected result length while fetching remote files\n");

		/* Read result set to local variables */
		memcpy(&chunkoff, PQgetvalue(res, 0, 1), sizeof(int64));
		chunkoff = pg_recvint64(chunkoff);
		chunksize = PQgetlength(res, 0, 2);

		filenamelen = PQgetlength(res, 0, 0);
		filename = pg_malloc(filenamelen + 1);
		memcpy(filename, PQgetvalue(res, 0, 0), filenamelen);
		filename[filenamelen] = '\0';

		chunk = PQgetvalue(res, 0, 2);

		/*
		 * If a file has been deleted on the source, remove it on the target
		 * as well.  Note that multiple unlink() calls may happen on the same
		 * file if multiple data chunks are associated with it, hence ignore
		 * unconditionally anything missing.  If this file is not a relation
		 * data file, then it has been already truncated when creating the
		 * file chunk list at the previous execution of the filemap.
		 */
		if (PQgetisnull(res, 0, 2))
		{
			pg_log(PG_DEBUG,
				   "received null value for chunk for file \"%s\", file has been deleted\n",
				   filename);
			remove_target_file(filename, true);
			pg_free(filename);
			PQclear(res);
			continue;
		}

		/*
		 * Separate step to keep platform-dependent format code out of
		 * translatable strings.
		 */
		snprintf(chunkoff_str, sizeof(chunkoff_str), INT64_FORMAT, chunkoff);
		pg_log(PG_DEBUG, "received chunk for file \"%s\", offset %s, size %d\n",
			   filename, chunkoff_str, chunksize);

		open_target_file(filename, false);

		write_target_range(chunk, chunkoff, chunksize);

		pg_free(filename);

		PQclear(res);
	}
}
예제 #5
0
/*----
 * Runs a query, which returns pieces of files from the remote source data
 * directory, and overwrites the corresponding parts of target files with
 * the received parts. The result set is expected to be of format:
 *
 * path		text	-- path in the data directory, e.g "base/1/123"
 * begin	int4	-- offset within the file
 * chunk	bytea	-- file content
 *----
 */
static void
receiveFileChunks(const char *sql)
{
	PGresult   *res;

	if (PQsendQueryParams(conn, sql, 0, NULL, NULL, NULL, NULL, 1) != 1)
		pg_fatal("could not send query: %s", PQerrorMessage(conn));

	pg_log(PG_DEBUG, "getting file chunks\n");

	if (PQsetSingleRowMode(conn) != 1)
		pg_fatal("could not set libpq connection to single row mode\n");

	while ((res = PQgetResult(conn)) != NULL)
	{
		char	   *filename;
		int			filenamelen;
		int			chunkoff;
		int			chunksize;
		char	   *chunk;

		switch (PQresultStatus(res))
		{
			case PGRES_SINGLE_TUPLE:
				break;

			case PGRES_TUPLES_OK:
				PQclear(res);
				continue;		/* final zero-row result */

			default:
				pg_fatal("unexpected result while fetching remote files: %s",
						 PQresultErrorMessage(res));
		}

		/* sanity check the result set */
		if (PQnfields(res) != 3 || PQntuples(res) != 1)
			pg_fatal("unexpected result set size while fetching remote files\n");

		if (PQftype(res, 0) != TEXTOID &&
			PQftype(res, 1) != INT4OID &&
			PQftype(res, 2) != BYTEAOID)
		{
			pg_fatal("unexpected data types in result set while fetching remote files: %u %u %u\n",
					 PQftype(res, 0), PQftype(res, 1), PQftype(res, 2));
		}

		if (PQfformat(res, 0) != 1 &&
			PQfformat(res, 1) != 1 &&
			PQfformat(res, 2) != 1)
		{
			pg_fatal("unexpected result format while fetching remote files\n");
		}

		if (PQgetisnull(res, 0, 0) ||
			PQgetisnull(res, 0, 1))
		{
			pg_fatal("unexpected null values in result while fetching remote files\n");
		}

		if (PQgetlength(res, 0, 1) != sizeof(int32))
			pg_fatal("unexpected result length while fetching remote files\n");

		/* Read result set to local variables */
		memcpy(&chunkoff, PQgetvalue(res, 0, 1), sizeof(int32));
		chunkoff = ntohl(chunkoff);
		chunksize = PQgetlength(res, 0, 2);

		filenamelen = PQgetlength(res, 0, 0);
		filename = pg_malloc(filenamelen + 1);
		memcpy(filename, PQgetvalue(res, 0, 0), filenamelen);
		filename[filenamelen] = '\0';

		chunk = PQgetvalue(res, 0, 2);

		/*
		 * It's possible that the file was deleted on remote side after we
		 * created the file map. In this case simply ignore it, as if it was
		 * not there in the first place, and move on.
		 */
		if (PQgetisnull(res, 0, 2))
		{
			pg_log(PG_DEBUG,
				   "received null value for chunk for file \"%s\", file has been deleted\n",
				   filename);
			pg_free(filename);
			PQclear(res);
			continue;
		}

		pg_log(PG_DEBUG, "received chunk for file \"%s\", offset %d, size %d\n",
			   filename, chunkoff, chunksize);

		open_target_file(filename, false);

		write_target_range(chunk, chunkoff, chunksize);

		pg_free(filename);

		PQclear(res);
	}
}
예제 #6
0
/*
 * Fetch all changed blocks from remote source data directory.
 */
void
libpq_executeFileMap(filemap_t *map)
{
	file_entry_t *entry;
	const char *sql;
	PGresult   *res;

	/*
	 * First create a temporary table, and load it with the blocks that
	 * we need to fetch.
	 */
	sql = "create temporary table fetchchunks(path text, begin int4, len int4);";
	res = PQexec(conn, sql);

	if (PQresultStatus(res) != PGRES_COMMAND_OK)
	{
		fprintf(stderr, "error creating temporary table: %s\n",
				PQresultErrorMessage(res));
		exit(1);
	}

	sql = "copy fetchchunks from stdin";
	res = PQexec(conn, sql);

	if (PQresultStatus(res) != PGRES_COPY_IN)
	{
		fprintf(stderr, "unexpected result while sending file list: %s\n",
				PQresultErrorMessage(res));
		exit(1);
	}

	for (entry = map->first; entry != NULL; entry = entry->next)
	{
		execute_pagemap(&entry->pagemap, entry->path);

		switch (entry->action)
		{
			case FILE_ACTION_NONE:
				/* ok, do nothing.. */
				break;

			case FILE_ACTION_COPY:
				/* Truncate the old file out of the way, if any */
				open_target_file(entry->path, true);
				copy_file_range(entry->path, 0, entry->newsize);
				break;

			case FILE_ACTION_REMOVE:
				remove_target_file(entry->path);
				break;

			case FILE_ACTION_TRUNCATE:
				truncate_target_file(entry->path, entry->newsize);
				break;

			case FILE_ACTION_COPY_TAIL:
				copy_file_range(entry->path, entry->oldsize, entry->newsize);
				break;

			case FILE_ACTION_CREATEDIR:
				create_target_dir(entry->path);
				break;

			case FILE_ACTION_REMOVEDIR:
				remove_target_dir(entry->path);
				break;
		}
	}

	if (PQputCopyEnd(conn, NULL) != 1)
	{
		fprintf(stderr, "error sending end-of-COPY: %s\n",
				PQerrorMessage(conn));
		exit(1);
	}

	while ((res = PQgetResult(conn)) != NULL)
	{
		if (PQresultStatus(res) != PGRES_COMMAND_OK)
		{
			fprintf(stderr, "unexpected result while sending file list: %s\n",
					PQresultErrorMessage(res));
			exit(1);
		}
	}

	/* Ok, we've sent the file list. Now receive the files */
	sql =
		"-- fetch all the blocks listed in the temp table.\n"
		"select path, begin, \n"
		"  pg_read_binary_file(path, begin, len) as chunk\n"
		"from fetchchunks\n";

	receiveFileChunks(sql);
}
예제 #7
0
/*
 * Runs a query, which returns pieces of files from the remote source data
 * directory, and overwrites the corresponding parts of target files with
 * the received parts. The result set is expected to be of format:
 *
 * path		text	-- path in the data directory, e.g "base/1/123"
 * begin	int4	-- offset within the file
 * chunk	bytea	-- file content
 *
 */
static void
receiveFileChunks(const char *sql)
{
	PGresult   *res;

	if (PQsendQueryParams(conn, sql, 0, NULL, NULL, NULL, NULL, 1) != 1)
	{
		fprintf(stderr, "could not send query: %s\n", PQerrorMessage(conn));
		exit(1);
	}

	if (verbose)
		fprintf(stderr, "getting chunks: %s\n", sql);

	if (PQsetSingleRowMode(conn) != 1)
	{
		fprintf(stderr, "could not set libpq connection to single row mode\n");
		exit(1);
	}

	if (verbose)
		fprintf(stderr, "sent query\n");

	while ((res = PQgetResult(conn)) != NULL)
	{
		char   *filename;
		int		filenamelen;
		int		chunkoff;
		int		chunksize;
		char   *chunk;

		switch(PQresultStatus(res))
		{
			case PGRES_SINGLE_TUPLE:
				break;

			case PGRES_TUPLES_OK:
				continue; /* final zero-row result */
			default:
				fprintf(stderr, "unexpected result while fetching remote files: %s\n",
						PQresultErrorMessage(res));
				exit(1);
		}

		/* sanity check the result set */
		if (!(PQnfields(res) == 3 && PQntuples(res) == 1))
		{
			fprintf(stderr, "unexpected result set size while fetching remote files\n");
			exit(1);
		}

		if (!(PQftype(res, 0) == TEXTOID && PQftype(res, 1) == INT4OID && PQftype(res, 2) == BYTEAOID))
		{
			fprintf(stderr, "unexpected data types in result set while fetching remote files: %u %u %u\n", PQftype(res, 0), PQftype(res, 1), PQftype(res, 2));
			exit(1);
		}
		if (!(PQfformat(res, 0) == 1 && PQfformat(res, 1) == 1 && PQfformat(res, 2) == 1))
		{
			fprintf(stderr, "unexpected result format while fetching remote files\n");
			exit(1);
		}

		if (!(!PQgetisnull(res, 0, 0) && !PQgetisnull(res, 0, 1) && !PQgetisnull(res, 0, 2) &&
			  PQgetlength(res, 0, 1) == sizeof(int32)))
		{
			fprintf(stderr, "unexpected result set while fetching remote files\n");
			exit(1);
		}

		/* Read result set to local variables */
		memcpy(&chunkoff, PQgetvalue(res, 0, 1), sizeof(int32));
		chunkoff = ntohl(chunkoff);
		chunksize = PQgetlength(res, 0, 2);

		filenamelen = PQgetlength(res, 0, 0);
		filename = pg_malloc(filenamelen + 1);
		memcpy(filename, PQgetvalue(res, 0, 0), filenamelen);
		filename[filenamelen] = '\0';

		chunk = PQgetvalue(res, 0, 2);

		if (verbose)
			fprintf(stderr, "received chunk for file \"%s\", off %d, len %d\n",
					filename, chunkoff, chunksize);

		open_target_file(filename, false);

		write_file_range(chunk, chunkoff, chunksize);
	}
}