/* * Copy a file from source to target, between 'begin' and 'end' offsets. * * If 'trunc' is true, any existing file with the same name is truncated. */ static void rewind_copy_file_range(const char *path, off_t begin, off_t end, bool trunc) { char buf[BLCKSZ]; char srcpath[MAXPGPATH]; int srcfd; snprintf(srcpath, sizeof(srcpath), "%s/%s", datadir_source, path); srcfd = open(srcpath, O_RDONLY | PG_BINARY, 0); if (srcfd < 0) pg_fatal("could not open source file \"%s\": %s\n", srcpath, strerror(errno)); if (lseek(srcfd, begin, SEEK_SET) == -1) pg_fatal("could not seek in source file: %s\n", strerror(errno)); open_target_file(path, trunc); while (end - begin > 0) { int readlen; int len; if (end - begin > sizeof(buf)) len = sizeof(buf); else len = end - begin; readlen = read(srcfd, buf, len); if (readlen < 0) pg_fatal("could not read file \"%s\": %s\n", srcpath, strerror(errno)); else if (readlen == 0) pg_fatal("unexpected EOF while reading file \"%s\"\n", srcpath); write_target_range(buf, begin, readlen); begin += readlen; } if (close(srcfd) != 0) pg_fatal("could not close file \"%s\": %s\n", srcpath, strerror(errno)); }
/*---- * Runs a query, which returns pieces of files from the remote source data * directory, and overwrites the corresponding parts of target files with * the received parts. The result set is expected to be of format: * * path text -- path in the data directory, e.g "base/1/123" * begin int8 -- offset within the file * chunk bytea -- file content *---- */ static void receiveFileChunks(const char *sql) { PGresult *res; if (PQsendQueryParams(conn, sql, 0, NULL, NULL, NULL, NULL, 1) != 1) pg_fatal("could not send query: %s", PQerrorMessage(conn)); pg_log(PG_DEBUG, "getting file chunks\n"); if (PQsetSingleRowMode(conn) != 1) pg_fatal("could not set libpq connection to single row mode\n"); while ((res = PQgetResult(conn)) != NULL) { char *filename; int filenamelen; int64 chunkoff; char chunkoff_str[32]; int chunksize; char *chunk; switch (PQresultStatus(res)) { case PGRES_SINGLE_TUPLE: break; case PGRES_TUPLES_OK: PQclear(res); continue; /* final zero-row result */ default: pg_fatal("unexpected result while fetching remote files: %s", PQresultErrorMessage(res)); } /* sanity check the result set */ if (PQnfields(res) != 3 || PQntuples(res) != 1) pg_fatal("unexpected result set size while fetching remote files\n"); if (PQftype(res, 0) != TEXTOID || PQftype(res, 1) != INT8OID || PQftype(res, 2) != BYTEAOID) { pg_fatal("unexpected data types in result set while fetching remote files: %u %u %u\n", PQftype(res, 0), PQftype(res, 1), PQftype(res, 2)); } if (PQfformat(res, 0) != 1 && PQfformat(res, 1) != 1 && PQfformat(res, 2) != 1) { pg_fatal("unexpected result format while fetching remote files\n"); } if (PQgetisnull(res, 0, 0) || PQgetisnull(res, 0, 1)) { pg_fatal("unexpected null values in result while fetching remote files\n"); } if (PQgetlength(res, 0, 1) != sizeof(int64)) pg_fatal("unexpected result length while fetching remote files\n"); /* Read result set to local variables */ memcpy(&chunkoff, PQgetvalue(res, 0, 1), sizeof(int64)); chunkoff = pg_recvint64(chunkoff); chunksize = PQgetlength(res, 0, 2); filenamelen = PQgetlength(res, 0, 0); filename = pg_malloc(filenamelen + 1); memcpy(filename, PQgetvalue(res, 0, 0), filenamelen); filename[filenamelen] = '\0'; chunk = PQgetvalue(res, 0, 2); /* * If a file has been deleted on the source, remove it on the target * as well. Note that multiple unlink() calls may happen on the same * file if multiple data chunks are associated with it, hence ignore * unconditionally anything missing. If this file is not a relation * data file, then it has been already truncated when creating the * file chunk list at the previous execution of the filemap. */ if (PQgetisnull(res, 0, 2)) { pg_log(PG_DEBUG, "received null value for chunk for file \"%s\", file has been deleted\n", filename); remove_target_file(filename, true); pg_free(filename); PQclear(res); continue; } /* * Separate step to keep platform-dependent format code out of * translatable strings. */ snprintf(chunkoff_str, sizeof(chunkoff_str), INT64_FORMAT, chunkoff); pg_log(PG_DEBUG, "received chunk for file \"%s\", offset %s, size %d\n", filename, chunkoff_str, chunksize); open_target_file(filename, false); write_target_range(chunk, chunkoff, chunksize); pg_free(filename); PQclear(res); } }
/*---- * Runs a query, which returns pieces of files from the remote source data * directory, and overwrites the corresponding parts of target files with * the received parts. The result set is expected to be of format: * * path text -- path in the data directory, e.g "base/1/123" * begin int4 -- offset within the file * chunk bytea -- file content *---- */ static void receiveFileChunks(const char *sql) { PGresult *res; if (PQsendQueryParams(conn, sql, 0, NULL, NULL, NULL, NULL, 1) != 1) pg_fatal("could not send query: %s", PQerrorMessage(conn)); pg_log(PG_DEBUG, "getting file chunks\n"); if (PQsetSingleRowMode(conn) != 1) pg_fatal("could not set libpq connection to single row mode\n"); while ((res = PQgetResult(conn)) != NULL) { char *filename; int filenamelen; int chunkoff; int chunksize; char *chunk; switch (PQresultStatus(res)) { case PGRES_SINGLE_TUPLE: break; case PGRES_TUPLES_OK: PQclear(res); continue; /* final zero-row result */ default: pg_fatal("unexpected result while fetching remote files: %s", PQresultErrorMessage(res)); } /* sanity check the result set */ if (PQnfields(res) != 3 || PQntuples(res) != 1) pg_fatal("unexpected result set size while fetching remote files\n"); if (PQftype(res, 0) != TEXTOID && PQftype(res, 1) != INT4OID && PQftype(res, 2) != BYTEAOID) { pg_fatal("unexpected data types in result set while fetching remote files: %u %u %u\n", PQftype(res, 0), PQftype(res, 1), PQftype(res, 2)); } if (PQfformat(res, 0) != 1 && PQfformat(res, 1) != 1 && PQfformat(res, 2) != 1) { pg_fatal("unexpected result format while fetching remote files\n"); } if (PQgetisnull(res, 0, 0) || PQgetisnull(res, 0, 1)) { pg_fatal("unexpected null values in result while fetching remote files\n"); } if (PQgetlength(res, 0, 1) != sizeof(int32)) pg_fatal("unexpected result length while fetching remote files\n"); /* Read result set to local variables */ memcpy(&chunkoff, PQgetvalue(res, 0, 1), sizeof(int32)); chunkoff = ntohl(chunkoff); chunksize = PQgetlength(res, 0, 2); filenamelen = PQgetlength(res, 0, 0); filename = pg_malloc(filenamelen + 1); memcpy(filename, PQgetvalue(res, 0, 0), filenamelen); filename[filenamelen] = '\0'; chunk = PQgetvalue(res, 0, 2); /* * It's possible that the file was deleted on remote side after we * created the file map. In this case simply ignore it, as if it was * not there in the first place, and move on. */ if (PQgetisnull(res, 0, 2)) { pg_log(PG_DEBUG, "received null value for chunk for file \"%s\", file has been deleted\n", filename); pg_free(filename); PQclear(res); continue; } pg_log(PG_DEBUG, "received chunk for file \"%s\", offset %d, size %d\n", filename, chunkoff, chunksize); open_target_file(filename, false); write_target_range(chunk, chunkoff, chunksize); pg_free(filename); PQclear(res); } }