/*
* new_9_0_populate_pg_largeobject_metadata()
* new >= 9.0, old <= 8.4
* 9.0 has a new pg_largeobject permission table
*/
void
new_9_0_populate_pg_largeobject_metadata(ClusterInfo *cluster, bool check_mode)
{
int dbnum;
FILE *script = NULL;
bool found = false;
char output_path[MAXPGPATH];
prep_status("Checking for large objects");
snprintf(output_path, sizeof(output_path), "pg_largeobject.sql");
for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++)
{
PGresult *res;
int i_count;
DbInfo *active_db = &cluster->dbarr.dbs[dbnum];
PGconn *conn = connectToServer(cluster, active_db->db_name);
/* find if there are any large objects */
res = executeQueryOrDie(conn,
"SELECT count(*) "
"FROM pg_catalog.pg_largeobject ");
i_count = PQfnumber(res, "count");
if (atoi(PQgetvalue(res, 0, i_count)) != 0)
{
found = true;
if (!check_mode)
{
if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL)
pg_fatal("could not open file \"%s\": %s\n", output_path, getErrorText(errno));
fprintf(script, "\\connect %s\n",
quote_identifier(active_db->db_name));
fprintf(script,
"SELECT pg_catalog.lo_create(t.loid)\n"
"FROM (SELECT DISTINCT loid FROM pg_catalog.pg_largeobject) AS t;\n");
}
}
PQclear(res);
PQfinish(conn);
}
if (script)
fclose(script);
if (found)
{
report_status(PG_WARNING, "warning");
if (check_mode)
pg_log(PG_WARNING, "\n"
"Your installation contains large objects. The new database has an\n"
"additional large object permission table. After upgrading, you will be\n"
"given a command to populate the pg_largeobject permission table with\n"
"default permissions.\n\n");
else
pg_log(PG_WARNING, "\n"
"Your installation contains large objects. The new database has an\n"
"additional large object permission table, so default permissions must be\n"
"defined for all large objects. The file\n"
" %s\n"
"when executed by psql by the database superuser will set the default\n"
"permissions.\n\n",
output_path);
}
else
check_ok();
}
/*
* old_9_3_check_for_line_data_type_usage()
* 9.3 -> 9.4
* Fully implement the 'line' data type in 9.4, which previously returned
* "not enabled" by default and was only functionally enabled with a
* compile-time switch; 9.4 "line" has different binary and text
* representation formats; checks tables and indexes.
*/
void
old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster)
{
int dbnum;
FILE *script = NULL;
bool found = false;
char output_path[MAXPGPATH];
prep_status("Checking for invalid \"line\" user columns");
snprintf(output_path, sizeof(output_path), "tables_using_line.txt");
for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++)
{
PGresult *res;
bool db_used = false;
int ntups;
int rowno;
int i_nspname,
i_relname,
i_attname;
DbInfo *active_db = &cluster->dbarr.dbs[dbnum];
PGconn *conn = connectToServer(cluster, active_db->db_name);
res = executeQueryOrDie(conn,
"SELECT n.nspname, c.relname, a.attname "
"FROM pg_catalog.pg_class c, "
" pg_catalog.pg_namespace n, "
" pg_catalog.pg_attribute a "
"WHERE c.oid = a.attrelid AND "
" NOT a.attisdropped AND "
" a.atttypid = 'pg_catalog.line'::pg_catalog.regtype AND "
" c.relnamespace = n.oid AND "
/* exclude possible orphaned temp tables */
" n.nspname !~ '^pg_temp_' AND "
" n.nspname !~ '^pg_toast_temp_' AND "
" n.nspname NOT IN ('pg_catalog', 'information_schema')");
ntups = PQntuples(res);
i_nspname = PQfnumber(res, "nspname");
i_relname = PQfnumber(res, "relname");
i_attname = PQfnumber(res, "attname");
for (rowno = 0; rowno < ntups; rowno++)
{
found = true;
if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL)
pg_fatal("could not open file \"%s\": %s\n", output_path, getErrorText(errno));
if (!db_used)
{
fprintf(script, "Database: %s\n", active_db->db_name);
db_used = true;
}
fprintf(script, " %s.%s.%s\n",
PQgetvalue(res, rowno, i_nspname),
PQgetvalue(res, rowno, i_relname),
PQgetvalue(res, rowno, i_attname));
}
PQclear(res);
PQfinish(conn);
}
if (script)
fclose(script);
if (found)
{
pg_log(PG_REPORT, "fatal\n");
pg_fatal("Your installation contains the \"line\" data type in user tables. This\n"
"data type changed its internal and input/output format between your old\n"
"and new clusters so this cluster cannot currently be upgraded. You can\n"
"remove the problem tables and restart the upgrade. A list of the problem\n"
"columns is in the file:\n"
" %s\n\n", output_path);
}
else
check_ok();
}
/*
* check_for_jsonb_9_4_usage()
*
* JSONB changed its storage format during 9.4 beta, so check for it.
*/
static void
check_for_jsonb_9_4_usage(ClusterInfo *cluster)
{
int dbnum;
FILE *script = NULL;
bool found = false;
char output_path[MAXPGPATH];
prep_status("Checking for incompatible \"jsonb\" data type");
snprintf(output_path, sizeof(output_path), "tables_using_jsonb.txt");
for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++)
{
PGresult *res;
bool db_used = false;
int ntups;
int rowno;
int i_nspname,
i_relname,
i_attname;
DbInfo *active_db = &cluster->dbarr.dbs[dbnum];
PGconn *conn = connectToServer(cluster, active_db->db_name);
/*
* While several relkinds don't store any data, e.g. views, they can
* be used to define data types of other columns, so we check all
* relkinds.
*/
res = executeQueryOrDie(conn,
"SELECT n.nspname, c.relname, a.attname "
"FROM pg_catalog.pg_class c, "
" pg_catalog.pg_namespace n, "
" pg_catalog.pg_attribute a "
"WHERE c.oid = a.attrelid AND "
" NOT a.attisdropped AND "
" a.atttypid = 'pg_catalog.jsonb'::pg_catalog.regtype AND "
" c.relnamespace = n.oid AND "
/* exclude possible orphaned temp tables */
" n.nspname !~ '^pg_temp_' AND "
" n.nspname NOT IN ('pg_catalog', 'information_schema')");
ntups = PQntuples(res);
i_nspname = PQfnumber(res, "nspname");
i_relname = PQfnumber(res, "relname");
i_attname = PQfnumber(res, "attname");
for (rowno = 0; rowno < ntups; rowno++)
{
found = true;
if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL)
pg_fatal("could not open file \"%s\": %s\n",
output_path, strerror(errno));
if (!db_used)
{
fprintf(script, "Database: %s\n", active_db->db_name);
db_used = true;
}
fprintf(script, " %s.%s.%s\n",
PQgetvalue(res, rowno, i_nspname),
PQgetvalue(res, rowno, i_relname),
PQgetvalue(res, rowno, i_attname));
}
PQclear(res);
PQfinish(conn);
}
if (script)
fclose(script);
if (found)
{
pg_log(PG_REPORT, "fatal\n");
pg_fatal("Your installation contains the \"jsonb\" data type in user tables.\n"
"The internal format of \"jsonb\" changed during 9.4 beta so this cluster cannot currently\n"
"be upgraded. You can remove the problem tables and restart the upgrade. A list\n"
"of the problem columns is in the file:\n"
" %s\n\n", output_path);
}
else
check_ok();
}
/*
* Callback for processing source file list.
*
* This is called once for every file in the source server. We decide what
* action needs to be taken for the file, depending on whether the file
* exists in the target and whether the size matches.
*/
void
process_source_file(const char *path, file_type_t type, size_t newsize,
const char *link_target)
{
bool exists;
char localpath[MAXPGPATH];
struct stat statbuf;
filemap_t *map = filemap;
file_action_t action = FILE_ACTION_NONE;
size_t oldsize = 0;
file_entry_t *entry;
Assert(map->array == NULL);
/*
* Completely ignore some special files in source and destination.
*/
if (strcmp(path, "postmaster.pid") == 0 ||
strcmp(path, "postmaster.opts") == 0)
return;
/*
* Skip temporary files, .../pgsql_tmp/... and .../pgsql_tmp.* in source.
* This has the effect that all temporary files in the destination will be
* removed.
*/
if (strstr(path, "/" PG_TEMP_FILE_PREFIX) != NULL)
return;
if (strstr(path, "/" PG_TEMP_FILES_DIR "/") != NULL)
return;
/*
* sanity check: a filename that looks like a data file better be a
* regular file
*/
if (type != FILE_TYPE_REGULAR && isRelDataFile(path))
pg_fatal("data file \"%s\" in source is not a regular file\n", path);
snprintf(localpath, sizeof(localpath), "%s/%s", datadir_target, path);
/* Does the corresponding file exist in the target data dir? */
if (lstat(localpath, &statbuf) < 0)
{
if (errno != ENOENT)
pg_fatal("could not stat file \"%s\": %s\n",
localpath, strerror(errno));
exists = false;
}
else
exists = true;
switch (type)
{
case FILE_TYPE_DIRECTORY:
if (exists && !S_ISDIR(statbuf.st_mode))
{
/* it's a directory in source, but not in target. Strange.. */
pg_fatal("\"%s\" is not a directory\n", localpath);
}
if (!exists)
action = FILE_ACTION_CREATE;
else
action = FILE_ACTION_NONE;
oldsize = 0;
break;
case FILE_TYPE_SYMLINK:
if (exists &&
#ifndef WIN32
!S_ISLNK(statbuf.st_mode)
#else
!pgwin32_is_junction(localpath)
#endif
)
{
/*
* It's a symbolic link in source, but not in target.
* Strange..
*/
pg_fatal("\"%s\" is not a symbolic link\n", localpath);
}
if (!exists)
action = FILE_ACTION_CREATE;
else
action = FILE_ACTION_NONE;
oldsize = 0;
break;
case FILE_TYPE_REGULAR:
if (exists && !S_ISREG(statbuf.st_mode))
pg_fatal("\"%s\" is not a regular file\n", localpath);
if (!exists || !isRelDataFile(path))
{
/*
* File exists in source, but not in target. Or it's a
* non-data file that we have no special processing for. Copy
* it in toto.
*
* An exception: PG_VERSIONs should be identical, but avoid
* overwriting it for paranoia.
*/
if (pg_str_endswith(path, "PG_VERSION"))
{
action = FILE_ACTION_NONE;
oldsize = statbuf.st_size;
}
else
{
action = FILE_ACTION_COPY;
oldsize = 0;
}
}
else
{
/*
* It's a data file that exists in both.
*
* If it's larger in target, we can truncate it. There will
* also be a WAL record of the truncation in the source
* system, so WAL replay would eventually truncate the target
* too, but we might as well do it now.
*
* If it's smaller in the target, it means that it has been
* truncated in the target, or enlarged in the source, or
* both. If it was truncated in the target, we need to copy
* the missing tail from the source system. If it was enlarged
* in the source system, there will be WAL records in the
* source system for the new blocks, so we wouldn't need to
* copy them here. But we don't know which scenario we're
* dealing with, and there's no harm in copying the missing
* blocks now, so do it now.
*
* If it's the same size, do nothing here. Any blocks modified
* in the target will be copied based on parsing the target
* system's WAL, and any blocks modified in the source will be
* updated after rewinding, when the source system's WAL is
* replayed.
*/
oldsize = statbuf.st_size;
if (oldsize < newsize)
action = FILE_ACTION_COPY_TAIL;
else if (oldsize > newsize)
action = FILE_ACTION_TRUNCATE;
else
action = FILE_ACTION_NONE;
}
break;
}
/* Create a new entry for this file */
entry = pg_malloc(sizeof(file_entry_t));
entry->path = pg_strdup(path);
entry->type = type;
entry->action = action;
entry->oldsize = oldsize;
entry->newsize = newsize;
entry->link_target = link_target ? pg_strdup(link_target) : NULL;
entry->next = NULL;
entry->pagemap.bitmap = NULL;
entry->pagemap.bitmapsize = 0;
entry->isrelfile = isRelDataFile(path);
if (map->last)
{
map->last->next = entry;
map->last = entry;
}
else
map->first = map->last = entry;
map->nlist++;
}
bool
start_postmaster(ClusterInfo *cluster, bool throw_error)
{
char cmd[MAXPGPATH * 4 + 1000];
PGconn *conn;
bool exit_hook_registered = false;
bool pg_ctl_return = false;
char socket_string[MAXPGPATH + 200];
if (!exit_hook_registered)
{
atexit(stop_postmaster_atexit);
exit_hook_registered = true;
}
socket_string[0] = '\0';
#ifdef HAVE_UNIX_SOCKETS
/* prevent TCP/IP connections, restrict socket access */
strcat(socket_string,
" -c listen_addresses='' -c unix_socket_permissions=0700");
/* Have a sockdir? Tell the postmaster. */
if (cluster->sockdir)
snprintf(socket_string + strlen(socket_string),
sizeof(socket_string) - strlen(socket_string),
" -c %s='%s'",
(GET_MAJOR_VERSION(cluster->major_version) < 903) ?
"unix_socket_directory" : "unix_socket_directories",
cluster->sockdir);
#endif
/*
* Since PG 9.1, we have used -b to disable autovacuum. For earlier
* releases, setting autovacuum=off disables cleanup vacuum and analyze,
* but freeze vacuums can still happen, so we set
* autovacuum_freeze_max_age to its maximum.
* (autovacuum_multixact_freeze_max_age was introduced after 9.1, so there
* is no need to set that.) We assume all datfrozenxid and relfrozenxid
* values are less than a gap of 2000000000 from the current xid counter,
* so autovacuum will not touch them.
*
* Turn off durability requirements to improve object creation speed, and
* we only modify the new cluster, so only use it there. If there is a
* crash, the new cluster has to be recreated anyway. fsync=off is a big
* win on ext4.
*/
snprintf(cmd, sizeof(cmd),
"\"%s/pg_ctl\" -w -l \"%s\" -D \"%s\" -o \"-p %d%s%s %s%s\" start",
cluster->bindir, SERVER_LOG_FILE, cluster->pgconfig, cluster->port,
(cluster->controldata.cat_ver >=
BINARY_UPGRADE_SERVER_FLAG_CAT_VER) ? " -b" :
" -c autovacuum=off -c autovacuum_freeze_max_age=2000000000",
(cluster == &new_cluster) ?
" -c synchronous_commit=off -c fsync=off -c full_page_writes=off" : "",
cluster->pgopts ? cluster->pgopts : "", socket_string);
/*
* Don't throw an error right away, let connecting throw the error because
* it might supply a reason for the failure.
*/
pg_ctl_return = exec_prog(SERVER_START_LOG_FILE,
/* pass both file names if they differ */
(strcmp(SERVER_LOG_FILE,
SERVER_START_LOG_FILE) != 0) ?
SERVER_LOG_FILE : NULL,
false,
"%s", cmd);
/* Did it fail and we are just testing if the server could be started? */
if (!pg_ctl_return && !throw_error)
return false;
/*
* We set this here to make sure atexit() shuts down the server, but only
* if we started the server successfully. We do it before checking for
* connectivity in case the server started but there is a connectivity
* failure. If pg_ctl did not return success, we will exit below.
*
* Pre-9.1 servers do not have PQping(), so we could be leaving the server
* running if authentication was misconfigured, so someday we might went
* to be more aggressive about doing server shutdowns even if pg_ctl
* fails, but now (2013-08-14) it seems prudent to be cautious. We don't
* want to shutdown a server that might have been accidentally started
* during the upgrade.
*/
if (pg_ctl_return)
os_info.running_cluster = cluster;
/*
* pg_ctl -w might have failed because the server couldn't be started, or
* there might have been a connection problem in _checking_ if the server
* has started. Therefore, even if pg_ctl failed, we continue and test
* for connectivity in case we get a connection reason for the failure.
*/
if ((conn = get_db_conn(cluster, "template1")) == NULL ||
PQstatus(conn) != CONNECTION_OK)
{
pg_log(PG_REPORT, "\nconnection to database failed: %s\n",
PQerrorMessage(conn));
if (conn)
PQfinish(conn);
pg_fatal("could not connect to %s postmaster started with the command:\n"
"%s\n",
CLUSTER_NAME(cluster), cmd);
}
PQfinish(conn);
/*
* If pg_ctl failed, and the connection didn't fail, and throw_error is
* enabled, fail now. This could happen if the server was already
* running.
*/
if (!pg_ctl_return)
pg_fatal("pg_ctl failed to start the %s server, or connection failed\n",
CLUSTER_NAME(cluster));
return true;
}
/*
* collect status from a completed worker child
*/
bool
reap_child(bool wait_for_child)
{
#ifndef WIN32
int work_status;
int ret;
#else
int thread_num;
DWORD res;
#endif
if (user_opts.jobs <= 1 || parallel_jobs == 0)
return false;
#ifndef WIN32
ret = waitpid(-1, &work_status, wait_for_child ? 0 : WNOHANG);
/* no children or, for WNOHANG, no dead children */
if (ret <= 0 || !WIFEXITED(work_status))
return false;
if (WEXITSTATUS(work_status) != 0)
pg_fatal("child worker exited abnormally: %s\n", strerror(errno));
#else
/* wait for one to finish */
thread_num = WaitForMultipleObjects(parallel_jobs, thread_handles,
false, wait_for_child ? INFINITE : 0);
if (thread_num == WAIT_TIMEOUT || thread_num == WAIT_FAILED)
return false;
/* compute thread index in active_threads */
thread_num -= WAIT_OBJECT_0;
/* get the result */
GetExitCodeThread(thread_handles[thread_num], &res);
if (res != 0)
pg_fatal("child worker exited abnormally: %s\n", strerror(errno));
/* dispose of handle to stop leaks */
CloseHandle(thread_handles[thread_num]);
/* Move last slot into dead child's position */
if (thread_num != parallel_jobs - 1)
{
void *tmp_args;
thread_handles[thread_num] = thread_handles[parallel_jobs - 1];
/*
* Move last active thead arg struct into the now-dead slot,
* and the now-dead slot to the end for reuse by the next thread.
* Though the thread struct is in use by another thread, we can
* safely swap the struct pointers within the array.
*/
tmp_args = cur_thread_args[thread_num];
cur_thread_args[thread_num] = cur_thread_args[parallel_jobs - 1];
cur_thread_args[parallel_jobs - 1] = tmp_args;
}
#endif
/* do this after job has been removed */
parallel_jobs--;
return true;
}
/*
* recursive part of traverse_datadir
*
* parentpath is the current subdirectory's path relative to datadir,
* or NULL at the top level.
*/
static void
recurse_dir(const char *datadir, const char *parentpath,
process_file_callback_t callback)
{
DIR *xldir;
struct dirent *xlde;
char fullparentpath[MAXPGPATH];
if (parentpath)
snprintf(fullparentpath, MAXPGPATH, "%s/%s", datadir, parentpath);
else
snprintf(fullparentpath, MAXPGPATH, "%s", datadir);
xldir = opendir(fullparentpath);
if (xldir == NULL)
pg_fatal("could not open directory \"%s\": %s\n",
fullparentpath, strerror(errno));
while (errno = 0, (xlde = readdir(xldir)) != NULL)
{
struct stat fst;
char fullpath[MAXPGPATH * 2];
char path[MAXPGPATH * 2];
if (strcmp(xlde->d_name, ".") == 0 ||
strcmp(xlde->d_name, "..") == 0)
continue;
snprintf(fullpath, sizeof(fullpath), "%s/%s", fullparentpath, xlde->d_name);
if (lstat(fullpath, &fst) < 0)
{
if (errno == ENOENT)
{
/*
* File doesn't exist anymore. This is ok, if the new master
* is running and the file was just removed. If it was a data
* file, there should be a WAL record of the removal. If it
* was something else, it couldn't have been anyway.
*
* TODO: But complain if we're processing the target dir!
*/
}
else
pg_fatal("could not stat file \"%s\": %s\n",
fullpath, strerror(errno));
}
if (parentpath)
snprintf(path, sizeof(path), "%s/%s", parentpath, xlde->d_name);
else
snprintf(path, sizeof(path), "%s", xlde->d_name);
if (S_ISREG(fst.st_mode))
callback(path, FILE_TYPE_REGULAR, fst.st_size, NULL);
else if (S_ISDIR(fst.st_mode))
{
callback(path, FILE_TYPE_DIRECTORY, 0, NULL);
/* recurse to handle subdirectories */
recurse_dir(datadir, path, callback);
}
#ifndef WIN32
else if (S_ISLNK(fst.st_mode))
#else
else if (pgwin32_is_junction(fullpath))
#endif
{
#if defined(HAVE_READLINK) || defined(WIN32)
char link_target[MAXPGPATH];
int len;
len = readlink(fullpath, link_target, sizeof(link_target));
if (len < 0)
pg_fatal("could not read symbolic link \"%s\": %s\n",
fullpath, strerror(errno));
if (len >= sizeof(link_target))
pg_fatal("symbolic link \"%s\" target is too long\n",
fullpath);
link_target[len] = '\0';
callback(path, FILE_TYPE_SYMLINK, 0, link_target);
/*
* If it's a symlink within pg_tblspc, we need to recurse into it,
* to process all the tablespaces. We also follow a symlink if
* it's for pg_wal. Symlinks elsewhere are ignored.
*/
if ((parentpath && strcmp(parentpath, "pg_tblspc") == 0) ||
strcmp(path, "pg_wal") == 0)
recurse_dir(datadir, path, callback);
#else
pg_fatal("\"%s\" is a symbolic link, but symbolic links are not supported on this platform\n",
fullpath);
#endif /* HAVE_READLINK */
}
}
if (errno)
pg_fatal("could not read directory \"%s\": %s\n",
fullparentpath, strerror(errno));
if (closedir(xldir))
pg_fatal("could not close directory \"%s\": %s\n",
fullparentpath, strerror(errno));
}
/*
* check_control_data()
*
* check to make sure the control data settings are compatible
*/
void
check_control_data(ControlData *oldctrl,
ControlData *newctrl)
{
if (oldctrl->align == 0 || oldctrl->align != newctrl->align)
pg_fatal("old and new pg_controldata alignments are invalid or do not match\n"
"Likely one cluster is a 32-bit install, the other 64-bit\n");
if (oldctrl->blocksz == 0 || oldctrl->blocksz != newctrl->blocksz)
pg_fatal("old and new pg_controldata block sizes are invalid or do not match\n");
if (oldctrl->largesz == 0 || oldctrl->largesz != newctrl->largesz)
pg_fatal("old and new pg_controldata maximum relation segement sizes are invalid or do not match\n");
if (oldctrl->walsz == 0 || oldctrl->walsz != newctrl->walsz)
pg_fatal("old and new pg_controldata WAL block sizes are invalid or do not match\n");
if (oldctrl->walseg == 0 || oldctrl->walseg != newctrl->walseg)
pg_fatal("old and new pg_controldata WAL segment sizes are invalid or do not match\n");
if (oldctrl->ident == 0 || oldctrl->ident != newctrl->ident)
pg_fatal("old and new pg_controldata maximum identifier lengths are invalid or do not match\n");
if (oldctrl->index == 0 || oldctrl->index != newctrl->index)
pg_fatal("old and new pg_controldata maximum indexed columns are invalid or do not match\n");
if (oldctrl->toast == 0 || oldctrl->toast != newctrl->toast)
pg_fatal("old and new pg_controldata maximum TOAST chunk sizes are invalid or do not match\n");
/* large_object added in 9.5, so it might not exist in the old cluster */
if (oldctrl->large_object != 0 &&
oldctrl->large_object != newctrl->large_object)
pg_fatal("old and new pg_controldata large-object chunk sizes are invalid or do not match\n");
if (oldctrl->date_is_int != newctrl->date_is_int)
pg_fatal("old and new pg_controldata date/time storage types do not match\n");
/*
* We might eventually allow upgrades from checksum to no-checksum
* clusters.
*/
if (oldctrl->data_checksum_version != newctrl->data_checksum_version)
{
pg_fatal("old and new pg_controldata checksum versions are invalid or do not match\n");
}
}
/*
* check_for_isn_and_int8_passing_mismatch()
*
* contrib/isn relies on data type int8, and in 8.4 int8 can now be passed
* by value. The schema dumps the CREATE TYPE PASSEDBYVALUE setting so
* it must match for the old and new servers.
*/
static void
check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster)
{
int dbnum;
FILE *script = NULL;
bool found = false;
char output_path[MAXPGPATH];
prep_status("Checking for contrib/isn with bigint-passing mismatch");
if (old_cluster.controldata.float8_pass_by_value ==
new_cluster.controldata.float8_pass_by_value)
{
/* no mismatch */
check_ok();
return;
}
snprintf(output_path, sizeof(output_path),
"contrib_isn_and_int8_pass_by_value.txt");
for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++)
{
PGresult *res;
bool db_used = false;
int ntups;
int rowno;
int i_nspname,
i_proname;
DbInfo *active_db = &cluster->dbarr.dbs[dbnum];
PGconn *conn = connectToServer(cluster, active_db->db_name);
/* Find any functions coming from contrib/isn */
res = executeQueryOrDie(conn,
"SELECT n.nspname, p.proname "
"FROM pg_catalog.pg_proc p, "
" pg_catalog.pg_namespace n "
"WHERE p.pronamespace = n.oid AND "
" p.probin = '$libdir/isn'");
ntups = PQntuples(res);
i_nspname = PQfnumber(res, "nspname");
i_proname = PQfnumber(res, "proname");
for (rowno = 0; rowno < ntups; rowno++)
{
found = true;
if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL)
pg_fatal("Could not open file \"%s\": %s\n",
output_path, getErrorText(errno));
if (!db_used)
{
fprintf(script, "Database: %s\n", active_db->db_name);
db_used = true;
}
fprintf(script, " %s.%s\n",
PQgetvalue(res, rowno, i_nspname),
PQgetvalue(res, rowno, i_proname));
}
PQclear(res);
PQfinish(conn);
}
if (script)
fclose(script);
if (found)
{
pg_log(PG_REPORT, "fatal\n");
pg_fatal("Your installation contains \"contrib/isn\" functions which rely on the\n"
"bigint data type. Your old and new clusters pass bigint values\n"
"differently so this cluster cannot currently be upgraded. You can\n"
"manually upgrade databases that use \"contrib/isn\" facilities and remove\n"
"\"contrib/isn\" from the old cluster and restart the upgrade. A list of\n"
"the problem functions is in the file:\n"
" %s\n\n", output_path);
}
else
check_ok();
}
/*
* check_for_reg_data_type_usage()
* pg_upgrade only preserves these system values:
* pg_class.oid
* pg_type.oid
* pg_enum.oid
*
* Many of the reg* data types reference system catalog info that is
* not preserved, and hence these data types cannot be used in user
* tables upgraded by pg_upgrade.
*/
static void
check_for_reg_data_type_usage(ClusterInfo *cluster)
{
int dbnum;
FILE *script = NULL;
bool found = false;
char output_path[MAXPGPATH];
prep_status("Checking for reg* system OID user data types");
snprintf(output_path, sizeof(output_path), "tables_using_reg.txt");
for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++)
{
PGresult *res;
bool db_used = false;
int ntups;
int rowno;
int i_nspname,
i_relname,
i_attname;
DbInfo *active_db = &cluster->dbarr.dbs[dbnum];
PGconn *conn = connectToServer(cluster, active_db->db_name);
/*
* While several relkinds don't store any data, e.g. views, they can
* be used to define data types of other columns, so we check all
* relkinds.
*/
res = executeQueryOrDie(conn,
"SELECT n.nspname, c.relname, a.attname "
"FROM pg_catalog.pg_class c, "
" pg_catalog.pg_namespace n, "
" pg_catalog.pg_attribute a "
"WHERE c.oid = a.attrelid AND "
" NOT a.attisdropped AND "
" a.atttypid IN ( "
" 'pg_catalog.regproc'::pg_catalog.regtype, "
" 'pg_catalog.regprocedure'::pg_catalog.regtype, "
" 'pg_catalog.regoper'::pg_catalog.regtype, "
" 'pg_catalog.regoperator'::pg_catalog.regtype, "
/* regclass.oid is preserved, so 'regclass' is OK */
/* regtype.oid is preserved, so 'regtype' is OK */
" 'pg_catalog.regconfig'::pg_catalog.regtype, "
" 'pg_catalog.regdictionary'::pg_catalog.regtype) AND "
" c.relnamespace = n.oid AND "
" n.nspname != 'pg_catalog' AND "
" n.nspname != 'information_schema'");
ntups = PQntuples(res);
i_nspname = PQfnumber(res, "nspname");
i_relname = PQfnumber(res, "relname");
i_attname = PQfnumber(res, "attname");
for (rowno = 0; rowno < ntups; rowno++)
{
found = true;
if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL)
pg_fatal("Could not open file \"%s\": %s\n",
output_path, getErrorText(errno));
if (!db_used)
{
fprintf(script, "Database: %s\n", active_db->db_name);
db_used = true;
}
fprintf(script, " %s.%s.%s\n",
PQgetvalue(res, rowno, i_nspname),
PQgetvalue(res, rowno, i_relname),
PQgetvalue(res, rowno, i_attname));
}
PQclear(res);
PQfinish(conn);
}
if (script)
fclose(script);
if (found)
{
pg_log(PG_REPORT, "fatal\n");
pg_fatal("Your installation contains one of the reg* data types in user tables.\n"
"These data types reference system OIDs that are not preserved by\n"
"pg_upgrade, so this cluster cannot currently be upgraded. You can\n"
"remove the problem tables and restart the upgrade. A list of the problem\n"
"columns is in the file:\n"
" %s\n\n", output_path);
}
else
check_ok();
}
/*
* create_script_for_old_cluster_deletion()
*
* This is particularly useful for tablespace deletion.
*/
void
create_script_for_old_cluster_deletion(char **deletion_script_file_name)
{
FILE *script = NULL;
int tblnum;
char old_cluster_pgdata[MAXPGPATH];
*deletion_script_file_name = psprintf("delete_old_cluster.%s", SCRIPT_EXT);
/*
* Some users (oddly) create tablespaces inside the cluster data
* directory. We can't create a proper old cluster delete script in that
* case.
*/
strlcpy(old_cluster_pgdata, old_cluster.pgdata, MAXPGPATH);
canonicalize_path(old_cluster_pgdata);
for (tblnum = 0; tblnum < os_info.num_old_tablespaces; tblnum++)
{
char old_tablespace_dir[MAXPGPATH];
strlcpy(old_tablespace_dir, os_info.old_tablespaces[tblnum], MAXPGPATH);
canonicalize_path(old_tablespace_dir);
if (path_is_prefix_of_path(old_cluster_pgdata, old_tablespace_dir))
{
/* Unlink file in case it is left over from a previous run. */
unlink(*deletion_script_file_name);
pg_free(*deletion_script_file_name);
*deletion_script_file_name = NULL;
return;
}
}
prep_status("Creating script to delete old cluster");
if ((script = fopen_priv(*deletion_script_file_name, "w")) == NULL)
pg_fatal("Could not open file \"%s\": %s\n",
*deletion_script_file_name, getErrorText(errno));
#ifndef WIN32
/* add shebang header */
fprintf(script, "#!/bin/sh\n\n");
#endif
/* delete old cluster's default tablespace */
fprintf(script, RMDIR_CMD " %s\n", fix_path_separator(old_cluster.pgdata));
/* delete old cluster's alternate tablespaces */
for (tblnum = 0; tblnum < os_info.num_old_tablespaces; tblnum++)
{
/*
* Do the old cluster's per-database directories share a directory
* with a new version-specific tablespace?
*/
if (strlen(old_cluster.tablespace_suffix) == 0)
{
/* delete per-database directories */
int dbnum;
fprintf(script, "\n");
/* remove PG_VERSION? */
if (GET_MAJOR_VERSION(old_cluster.major_version) <= 804)
fprintf(script, RM_CMD " %s%cPG_VERSION\n",
fix_path_separator(os_info.old_tablespaces[tblnum]),
PATH_SEPARATOR);
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
fprintf(script, RMDIR_CMD " %s%c%d\n",
fix_path_separator(os_info.old_tablespaces[tblnum]),
PATH_SEPARATOR, old_cluster.dbarr.dbs[dbnum].db_oid);
}
else
{
char *suffix_path = pg_strdup(old_cluster.tablespace_suffix);
/*
* Simply delete the tablespace directory, which might be ".old"
* or a version-specific subdirectory.
*/
fprintf(script, RMDIR_CMD " %s%s\n",
fix_path_separator(os_info.old_tablespaces[tblnum]),
fix_path_separator(suffix_path));
pfree(suffix_path);
}
}
fclose(script);
#ifndef WIN32
if (chmod(*deletion_script_file_name, S_IRWXU) != 0)
pg_fatal("Could not add execute permission to file \"%s\": %s\n",
*deletion_script_file_name, getErrorText(errno));
#endif
check_ok();
}
/*
* create_script_for_cluster_analyze()
*
* This incrementally generates better optimizer statistics
*/
void
create_script_for_cluster_analyze(char **analyze_script_file_name)
{
FILE *script = NULL;
char *user_specification = "";
prep_status("Creating script to analyze new cluster");
if (os_info.user_specified)
user_specification = psprintf("-U \"%s\" ", os_info.user);
*analyze_script_file_name = psprintf("analyze_new_cluster.%s", SCRIPT_EXT);
if ((script = fopen_priv(*analyze_script_file_name, "w")) == NULL)
pg_fatal("Could not open file \"%s\": %s\n",
*analyze_script_file_name, getErrorText(errno));
#ifndef WIN32
/* add shebang header */
fprintf(script, "#!/bin/sh\n\n");
#else
/* suppress command echoing */
fprintf(script, "@echo off\n");
#endif
fprintf(script, "echo %sThis script will generate minimal optimizer statistics rapidly%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %sso your system is usable, and then gather statistics twice more%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %swith increasing accuracy. When it is done, your system will%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %shave the default level of optimizer statistics.%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo%s\n\n", ECHO_BLANK);
fprintf(script, "echo %sIf you have used ALTER TABLE to modify the statistics target for%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %sany tables, you might want to remove them and restore them after%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %srunning this script because they will delay fast statistics generation.%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo%s\n\n", ECHO_BLANK);
fprintf(script, "echo %sIf you would like default statistics as quickly as possible, cancel%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %sthis script and run:%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %s \"%s/vacuumdb\" %s--all %s%s\n", ECHO_QUOTE,
new_cluster.bindir, user_specification,
/* Did we copy the free space files? */
(GET_MAJOR_VERSION(old_cluster.major_version) >= 804) ?
"--analyze-only" : "--analyze", ECHO_QUOTE);
fprintf(script, "echo%s\n\n", ECHO_BLANK);
fprintf(script, "\"%s/vacuumdb\" %s--all --analyze-in-stages\n",
new_cluster.bindir, user_specification);
/* Did we copy the free space files? */
if (GET_MAJOR_VERSION(old_cluster.major_version) < 804)
fprintf(script, "\"%s/vacuumdb\" %s--all\n", new_cluster.bindir,
user_specification);
fprintf(script, "echo%s\n\n", ECHO_BLANK);
fprintf(script, "echo %sDone%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fclose(script);
#ifndef WIN32
if (chmod(*analyze_script_file_name, S_IRWXU) != 0)
pg_fatal("Could not add execute permission to file \"%s\": %s\n",
*analyze_script_file_name, getErrorText(errno));
#endif
if (os_info.user_specified)
pg_free(user_specification);
check_ok();
}
/*
* gen_db_file_maps()
*
* generates a database mapping from "old_db" to "new_db".
*
* Returns a malloc'ed array of mappings. The length of the array
* is returned into *nmaps.
*/
FileNameMap *
gen_db_file_maps(DbInfo *old_db, DbInfo *new_db,
int *nmaps,
const char *old_pgdata, const char *new_pgdata)
{
FileNameMap *maps;
int old_relnum,
new_relnum;
int num_maps = 0;
bool all_matched = true;
/* There will certainly not be more mappings than there are old rels */
maps = (FileNameMap *) pg_malloc(sizeof(FileNameMap) *
old_db->rel_arr.nrels);
/*
* Each of the RelInfo arrays should be sorted by OID. Scan through them
* and match them up. If we fail to match everything, we'll abort, but
* first print as much info as we can about mismatches.
*/
old_relnum = new_relnum = 0;
while (old_relnum < old_db->rel_arr.nrels ||
new_relnum < new_db->rel_arr.nrels)
{
RelInfo *old_rel = (old_relnum < old_db->rel_arr.nrels) ?
&old_db->rel_arr.rels[old_relnum] : NULL;
RelInfo *new_rel = (new_relnum < new_db->rel_arr.nrels) ?
&new_db->rel_arr.rels[new_relnum] : NULL;
/* handle running off one array before the other */
if (!new_rel)
{
/*
* old_rel is unmatched. This should never happen, because we
* force new rels to have TOAST tables if the old one did.
*/
report_unmatched_relation(old_rel, old_db, false);
all_matched = false;
old_relnum++;
continue;
}
if (!old_rel)
{
/*
* new_rel is unmatched. This shouldn't really happen either, but
* if it's a TOAST table, we can ignore it and continue
* processing, assuming that the new server made a TOAST table
* that wasn't needed.
*/
if (strcmp(new_rel->nspname, "pg_toast") != 0)
{
report_unmatched_relation(new_rel, new_db, true);
all_matched = false;
}
new_relnum++;
continue;
}
/* check for mismatched OID */
if (old_rel->reloid < new_rel->reloid)
{
/* old_rel is unmatched, see comment above */
report_unmatched_relation(old_rel, old_db, false);
all_matched = false;
old_relnum++;
continue;
}
else if (old_rel->reloid > new_rel->reloid)
{
/* new_rel is unmatched, see comment above */
if (strcmp(new_rel->nspname, "pg_toast") != 0)
{
report_unmatched_relation(new_rel, new_db, true);
all_matched = false;
}
new_relnum++;
continue;
}
/*
* Verify that rels of same OID have same name. The namespace name
* should always match, but the relname might not match for TOAST
* tables (and, therefore, their indexes).
*
* TOAST table names initially match the heap pg_class oid, but
* pre-9.0 they can change during certain commands such as CLUSTER, so
* don't insist on a match if old cluster is < 9.0.
*/
if (strcmp(old_rel->nspname, new_rel->nspname) != 0 ||
(strcmp(old_rel->relname, new_rel->relname) != 0 &&
(GET_MAJOR_VERSION(old_cluster.major_version) >= 900 ||
strcmp(old_rel->nspname, "pg_toast") != 0)))
{
pg_log(PG_WARNING, "Relation names for OID %u in database \"%s\" do not match: "
"old name \"%s.%s\", new name \"%s.%s\"\n",
old_rel->reloid, old_db->db_name,
old_rel->nspname, old_rel->relname,
new_rel->nspname, new_rel->relname);
all_matched = false;
old_relnum++;
new_relnum++;
continue;
}
/* OK, create a mapping entry */
create_rel_filename_map(old_pgdata, new_pgdata, old_db, new_db,
old_rel, new_rel, maps + num_maps);
num_maps++;
old_relnum++;
new_relnum++;
}
if (!all_matched)
pg_fatal("Failed to match up old and new tables in database \"%s\"\n",
old_db->db_name);
*nmaps = num_maps;
return maps;
}
/*
* exec_prog()
* Execute an external program with stdout/stderr redirected, and report
* errors
*
* Formats a command from the given argument list, logs it to the log file,
* and attempts to execute that command. If the command executes
* successfully, exec_prog() returns true.
*
* If the command fails, an error message is saved to the specified log_file.
* If throw_error is true, this raises a PG_FATAL error and pg_upgrade
* terminates; otherwise it is just reported as PG_REPORT and exec_prog()
* returns false.
*
* The code requires it be called first from the primary thread on Windows.
*/
bool
exec_prog(const char *log_file, const char *opt_log_file,
bool throw_error, const char *fmt,...)
{
int result = 0;
int written;
#define MAXCMDLEN (2 * MAXPGPATH)
char cmd[MAXCMDLEN];
FILE *log;
va_list ap;
#ifdef WIN32
static DWORD mainThreadId = 0;
/* We assume we are called from the primary thread first */
if (mainThreadId == 0)
mainThreadId = GetCurrentThreadId();
#endif
written = 0;
va_start(ap, fmt);
written += vsnprintf(cmd + written, MAXCMDLEN - written, fmt, ap);
va_end(ap);
if (written >= MAXCMDLEN)
pg_fatal("command too long\n");
written += snprintf(cmd + written, MAXCMDLEN - written,
" >> \"%s\" 2>&1", log_file);
if (written >= MAXCMDLEN)
pg_fatal("command too long\n");
pg_log(PG_VERBOSE, "%s\n", cmd);
#ifdef WIN32
/*
* For some reason, Windows issues a file-in-use error if we write data to
* the log file from a non-primary thread just before we create a
* subprocess that also writes to the same log file. One fix is to sleep
* for 100ms. A cleaner fix is to write to the log file _after_ the
* subprocess has completed, so we do this only when writing from a
* non-primary thread. fflush(), running system() twice, and pre-creating
* the file do not see to help.
*/
if (mainThreadId != GetCurrentThreadId())
result = system(cmd);
#endif
log = fopen(log_file, "a");
#ifdef WIN32
{
/*
* "pg_ctl -w stop" might have reported that the server has stopped
* because the postmaster.pid file has been removed, but "pg_ctl -w
* start" might still be in the process of closing and might still be
* holding its stdout and -l log file descriptors open. Therefore,
* try to open the log file a few more times.
*/
int iter;
for (iter = 0; iter < 4 && log == NULL; iter++)
{
pg_usleep(1000000); /* 1 sec */
log = fopen(log_file, "a");
}
}
#endif
if (log == NULL)
pg_fatal("cannot write to log file %s\n", log_file);
#ifdef WIN32
/* Are we printing "command:" before its output? */
if (mainThreadId == GetCurrentThreadId())
fprintf(log, "\n\n");
#endif
fprintf(log, "command: %s\n", cmd);
#ifdef WIN32
/* Are we printing "command:" after its output? */
if (mainThreadId != GetCurrentThreadId())
fprintf(log, "\n\n");
#endif
/*
* In Windows, we must close the log file at this point so the file is not
* open while the command is running, or we get a share violation.
*/
fclose(log);
#ifdef WIN32
/* see comment above */
if (mainThreadId == GetCurrentThreadId())
#endif
result = system(cmd);
if (result != 0)
{
/* we might be in on a progress status line, so go to the next line */
report_status(PG_REPORT, "\n*failure*");
fflush(stdout);
pg_log(PG_VERBOSE, "There were problems executing \"%s\"\n", cmd);
if (opt_log_file)
pg_log(throw_error ? PG_FATAL : PG_REPORT,
"Consult the last few lines of \"%s\" or \"%s\" for\n"
"the probable cause of the failure.\n",
log_file, opt_log_file);
else
pg_log(throw_error ? PG_FATAL : PG_REPORT,
"Consult the last few lines of \"%s\" for\n"
"the probable cause of the failure.\n",
log_file);
}
#ifndef WIN32
/*
* We can't do this on Windows because it will keep the "pg_ctl start"
* output filename open until the server stops, so we do the \n\n above on
* that platform. We use a unique filename for "pg_ctl start" that is
* never reused while the server is running, so it works fine. We could
* log these commands to a third file, but that just adds complexity.
*/
if ((log = fopen(log_file, "a")) == NULL)
pg_fatal("cannot write to log file %s\n", log_file);
fprintf(log, "\n\n");
fclose(log);
#endif
return result == 0;
}
/*
* parseCommandLine()
*
* Parses the command line (argc, argv[]) and loads structures
*/
void
parseCommandLine(int argc, char *argv[])
{
static struct option long_options[] = {
{"old-datadir", required_argument, NULL, 'd'},
{"new-datadir", required_argument, NULL, 'D'},
{"old-bindir", required_argument, NULL, 'b'},
{"new-bindir", required_argument, NULL, 'B'},
{"old-options", required_argument, NULL, 'o'},
{"new-options", required_argument, NULL, 'O'},
{"old-port", required_argument, NULL, 'p'},
{"new-port", required_argument, NULL, 'P'},
{"username", required_argument, NULL, 'U'},
{"check", no_argument, NULL, 'c'},
{"link", no_argument, NULL, 'k'},
{"retain", no_argument, NULL, 'r'},
{"jobs", required_argument, NULL, 'j'},
{"verbose", no_argument, NULL, 'v'},
{NULL, 0, NULL, 0}
};
int option; /* Command line option */
int optindex = 0; /* used by getopt_long */
int os_user_effective_id;
FILE *fp;
char **filename;
time_t run_time = time(NULL);
user_opts.transfer_mode = TRANSFER_MODE_COPY;
os_info.progname = get_progname(argv[0]);
/* Process libpq env. variables; load values here for usage() output */
old_cluster.port = getenv("PGPORTOLD") ? atoi(getenv("PGPORTOLD")) : DEF_PGUPORT;
new_cluster.port = getenv("PGPORTNEW") ? atoi(getenv("PGPORTNEW")) : DEF_PGUPORT;
os_user_effective_id = get_user_info(&os_info.user);
/* we override just the database user name; we got the OS id above */
if (getenv("PGUSER"))
{
pg_free(os_info.user);
/* must save value, getenv()'s pointer is not stable */
os_info.user = pg_strdup(getenv("PGUSER"));
}
if (argc > 1)
{
if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
{
usage();
exit(0);
}
if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
{
puts("pg_upgrade (PostgreSQL) " PG_VERSION);
exit(0);
}
}
/* Allow help and version to be run as root, so do the test here. */
if (os_user_effective_id == 0)
pg_fatal("%s: cannot be run as root\n", os_info.progname);
if ((log_opts.internal = fopen_priv(INTERNAL_LOG_FILE, "a")) == NULL)
pg_fatal("cannot write to log file %s\n", INTERNAL_LOG_FILE);
while ((option = getopt_long(argc, argv, "d:D:b:B:cj:ko:O:p:P:rU:v",
long_options, &optindex)) != -1)
{
switch (option)
{
case 'b':
old_cluster.bindir = pg_strdup(optarg);
break;
case 'B':
new_cluster.bindir = pg_strdup(optarg);
break;
case 'c':
user_opts.check = true;
break;
case 'd':
old_cluster.pgdata = pg_strdup(optarg);
old_cluster.pgconfig = pg_strdup(optarg);
break;
case 'D':
new_cluster.pgdata = pg_strdup(optarg);
new_cluster.pgconfig = pg_strdup(optarg);
break;
case 'j':
user_opts.jobs = atoi(optarg);
break;
case 'k':
user_opts.transfer_mode = TRANSFER_MODE_LINK;
break;
case 'o':
old_cluster.pgopts = pg_strdup(optarg);
break;
case 'O':
new_cluster.pgopts = pg_strdup(optarg);
break;
/*
* Someday, the port number option could be removed and passed
* using -o/-O, but that requires postmaster -C to be
* supported on all old/new versions.
*/
case 'p':
if ((old_cluster.port = atoi(optarg)) <= 0)
{
pg_fatal("invalid old port number\n");
exit(1);
}
break;
case 'P':
if ((new_cluster.port = atoi(optarg)) <= 0)
{
pg_fatal("invalid new port number\n");
exit(1);
}
break;
case 'r':
log_opts.retain = true;
break;
case 'U':
pg_free(os_info.user);
os_info.user = pg_strdup(optarg);
os_info.user_specified = true;
/*
* Push the user name into the environment so pre-9.1
* pg_ctl/libpq uses it.
*/
pg_putenv("PGUSER", os_info.user);
break;
case 'v':
pg_log(PG_REPORT, "Running in verbose mode\n");
log_opts.verbose = true;
break;
default:
pg_fatal("Try \"%s --help\" for more information.\n",
os_info.progname);
break;
}
}
/* label start of upgrade in logfiles */
for (filename = output_files; *filename != NULL; filename++)
{
if ((fp = fopen_priv(*filename, "a")) == NULL)
pg_fatal("cannot write to log file %s\n", *filename);
/* Start with newline because we might be appending to a file. */
fprintf(fp, "\n"
"-----------------------------------------------------------------\n"
" pg_upgrade run on %s"
"-----------------------------------------------------------------\n\n",
ctime(&run_time));
fclose(fp);
}
/* Turn off read-only mode; add prefix to PGOPTIONS? */
if (getenv("PGOPTIONS"))
{
char *pgoptions = psprintf("%s %s", FIX_DEFAULT_READ_ONLY,
getenv("PGOPTIONS"));
pg_putenv("PGOPTIONS", pgoptions);
pfree(pgoptions);
}
else
pg_putenv("PGOPTIONS", FIX_DEFAULT_READ_ONLY);
/* Get values from env if not already set */
check_required_directory(&old_cluster.bindir, NULL, "PGBINOLD", "-b",
"old cluster binaries reside");
check_required_directory(&new_cluster.bindir, NULL, "PGBINNEW", "-B",
"new cluster binaries reside");
check_required_directory(&old_cluster.pgdata, &old_cluster.pgconfig,
"PGDATAOLD", "-d", "old cluster data resides");
check_required_directory(&new_cluster.pgdata, &new_cluster.pgconfig,
"PGDATANEW", "-D", "new cluster data resides");
}
/*
* get_control_data()
*
* gets pg_control information in "ctrl". Assumes that bindir and
* datadir are valid absolute paths to postgresql bin and pgdata
* directories respectively *and* pg_resetxlog is version compatible
* with datadir. The main purpose of this function is to get pg_control
* data in a version independent manner.
*
* The approach taken here is to invoke pg_resetxlog with -n option
* and then pipe its output. With little string parsing we get the
* pg_control data. pg_resetxlog cannot be run while the server is running
* so we use pg_controldata; pg_controldata doesn't provide all the fields
* we need to actually perform the upgrade, but it provides enough for
* check mode. We do not implement pg_resetxlog -n because it is hard to
* return valid xid data for a running server.
*/
void
get_control_data(ClusterInfo *cluster, bool live_check)
{
char cmd[MAXPGPATH];
char bufin[MAX_STRING];
FILE *output;
char *p;
bool got_xid = false;
bool got_oid = false;
bool got_nextxlogfile = false;
bool got_multi = false;
bool got_mxoff = false;
bool got_oldestmulti = false;
bool got_log_id = false;
bool got_log_seg = false;
bool got_tli = false;
bool got_align = false;
bool got_blocksz = false;
bool got_largesz = false;
bool got_walsz = false;
bool got_walseg = false;
bool got_ident = false;
bool got_index = false;
bool got_toast = false;
bool got_large_object = false;
bool got_date_is_int = false;
bool got_float8_pass_by_value = false;
bool got_data_checksum_version = false;
char *lc_collate = NULL;
char *lc_ctype = NULL;
char *lc_monetary = NULL;
char *lc_numeric = NULL;
char *lc_time = NULL;
char *lang = NULL;
char *language = NULL;
char *lc_all = NULL;
char *lc_messages = NULL;
uint32 logid = 0;
uint32 segno = 0;
uint32 tli = 0;
/*
* Because we test the pg_resetxlog output as strings, it has to be in
* English. Copied from pg_regress.c.
*/
if (getenv("LC_COLLATE"))
lc_collate = pg_strdup(getenv("LC_COLLATE"));
if (getenv("LC_CTYPE"))
lc_ctype = pg_strdup(getenv("LC_CTYPE"));
if (getenv("LC_MONETARY"))
lc_monetary = pg_strdup(getenv("LC_MONETARY"));
if (getenv("LC_NUMERIC"))
lc_numeric = pg_strdup(getenv("LC_NUMERIC"));
if (getenv("LC_TIME"))
lc_time = pg_strdup(getenv("LC_TIME"));
if (getenv("LANG"))
lang = pg_strdup(getenv("LANG"));
if (getenv("LANGUAGE"))
language = pg_strdup(getenv("LANGUAGE"));
if (getenv("LC_ALL"))
lc_all = pg_strdup(getenv("LC_ALL"));
if (getenv("LC_MESSAGES"))
lc_messages = pg_strdup(getenv("LC_MESSAGES"));
pg_putenv("LC_COLLATE", NULL);
pg_putenv("LC_CTYPE", NULL);
pg_putenv("LC_MONETARY", NULL);
pg_putenv("LC_NUMERIC", NULL);
pg_putenv("LC_TIME", NULL);
pg_putenv("LANG",
#ifndef WIN32
NULL);
#else
/* On Windows the default locale cannot be English, so force it */
"en");
#endif
pg_putenv("LANGUAGE", NULL);
pg_putenv("LC_ALL", NULL);
pg_putenv("LC_MESSAGES", "C");
snprintf(cmd, sizeof(cmd), "\"%s/%s \"%s\"",
cluster->bindir,
live_check ? "pg_controldata\"" : "pg_resetxlog\" -n",
cluster->pgdata);
fflush(stdout);
fflush(stderr);
if ((output = popen(cmd, "r")) == NULL)
pg_fatal("Could not get control data using %s: %s\n",
cmd, getErrorText(errno));
/* Only in <= 9.2 */
if (GET_MAJOR_VERSION(cluster->major_version) <= 902)
{
cluster->controldata.data_checksum_version = 0;
got_data_checksum_version = true;
}
/* we have the result of cmd in "output". so parse it line by line now */
while (fgets(bufin, sizeof(bufin), output))
{
pg_log(PG_VERBOSE, "%s", bufin);
if ((p = strstr(bufin, "pg_control version number:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: pg_resetxlog problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.ctrl_ver = str2uint(p);
}
else if ((p = strstr(bufin, "Catalog version number:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.cat_ver = str2uint(p);
}
else if ((p = strstr(bufin, "First log segment after reset:")) != NULL)
{
/* Skip the colon and any whitespace after it */
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p = strpbrk(p, "01234567890ABCDEF");
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
/* Make sure it looks like a valid WAL file name */
if (strspn(p, "0123456789ABCDEF") != 24)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
strlcpy(cluster->controldata.nextxlogfile, p, 25);
got_nextxlogfile = true;
}
else if ((p = strstr(bufin, "First log file ID after reset:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
logid = str2uint(p);
got_log_id = true;
}
else if ((p = strstr(bufin, "First log file segment after reset:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
segno = str2uint(p);
got_log_seg = true;
}
else if ((p = strstr(bufin, "Latest checkpoint's TimeLineID:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.chkpnt_tli = str2uint(p);
got_tli = true;
}
else if ((p = strstr(bufin, "Latest checkpoint's NextXID:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.chkpnt_nxtepoch = str2uint(p);
p = strchr(p, '/');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove '/' char */
cluster->controldata.chkpnt_nxtxid = str2uint(p);
got_xid = true;
}
else if ((p = strstr(bufin, "Latest checkpoint's NextOID:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.chkpnt_nxtoid = str2uint(p);
got_oid = true;
}
else if ((p = strstr(bufin, "Latest checkpoint's NextMultiXactId:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.chkpnt_nxtmulti = str2uint(p);
got_multi = true;
}
else if ((p = strstr(bufin, "Latest checkpoint's oldestMultiXid:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.chkpnt_oldstMulti = str2uint(p);
got_oldestmulti = true;
}
else if ((p = strstr(bufin, "Latest checkpoint's NextMultiOffset:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.chkpnt_nxtmxoff = str2uint(p);
got_mxoff = true;
}
else if ((p = strstr(bufin, "Maximum data alignment:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.align = str2uint(p);
got_align = true;
}
else if ((p = strstr(bufin, "Database block size:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.blocksz = str2uint(p);
got_blocksz = true;
}
else if ((p = strstr(bufin, "Blocks per segment of large relation:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.largesz = str2uint(p);
got_largesz = true;
}
else if ((p = strstr(bufin, "WAL block size:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.walsz = str2uint(p);
got_walsz = true;
}
else if ((p = strstr(bufin, "Bytes per WAL segment:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.walseg = str2uint(p);
got_walseg = true;
}
else if ((p = strstr(bufin, "Maximum length of identifiers:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.ident = str2uint(p);
got_ident = true;
}
else if ((p = strstr(bufin, "Maximum columns in an index:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.index = str2uint(p);
got_index = true;
}
else if ((p = strstr(bufin, "Maximum size of a TOAST chunk:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.toast = str2uint(p);
got_toast = true;
}
else if ((p = strstr(bufin, "Size of a large-object chunk:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.large_object = str2uint(p);
got_large_object = true;
}
else if ((p = strstr(bufin, "Date/time type storage:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
cluster->controldata.date_is_int = strstr(p, "64-bit integers") != NULL;
got_date_is_int = true;
}
else if ((p = strstr(bufin, "Float8 argument passing:")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
/* used later for contrib check */
cluster->controldata.float8_pass_by_value = strstr(p, "by value") != NULL;
got_float8_pass_by_value = true;
}
else if ((p = strstr(bufin, "checksum")) != NULL)
{
p = strchr(p, ':');
if (p == NULL || strlen(p) <= 1)
pg_fatal("%d: controldata retrieval problem\n", __LINE__);
p++; /* remove ':' char */
/* used later for contrib check */
cluster->controldata.data_checksum_version = str2uint(p);
got_data_checksum_version = true;
}
}
if (output)
pclose(output);
/*
* Restore environment variables
*/
pg_putenv("LC_COLLATE", lc_collate);
pg_putenv("LC_CTYPE", lc_ctype);
pg_putenv("LC_MONETARY", lc_monetary);
pg_putenv("LC_NUMERIC", lc_numeric);
pg_putenv("LC_TIME", lc_time);
pg_putenv("LANG", lang);
pg_putenv("LANGUAGE", language);
pg_putenv("LC_ALL", lc_all);
pg_putenv("LC_MESSAGES", lc_messages);
pg_free(lc_collate);
pg_free(lc_ctype);
pg_free(lc_monetary);
pg_free(lc_numeric);
pg_free(lc_time);
pg_free(lang);
pg_free(language);
pg_free(lc_all);
pg_free(lc_messages);
/*
* Before 9.3, pg_resetxlog reported the xlogid and segno of the first log
* file after reset as separate lines. Starting with 9.3, it reports the
* WAL file name. If the old cluster is older than 9.3, we construct the
* WAL file name from the xlogid and segno.
*/
if (GET_MAJOR_VERSION(cluster->major_version) <= 902)
{
if (got_log_id && got_log_seg)
{
snprintf(cluster->controldata.nextxlogfile, 25, "%08X%08X%08X",
tli, logid, segno);
got_nextxlogfile = true;
}
}
/* verify that we got all the mandatory pg_control data */
if (!got_xid || !got_oid ||
!got_multi || !got_mxoff ||
(!got_oldestmulti &&
cluster->controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER) ||
(!live_check && !got_nextxlogfile) ||
!got_tli ||
!got_align || !got_blocksz || !got_largesz || !got_walsz ||
!got_walseg || !got_ident || !got_index || !got_toast ||
(!got_large_object &&
cluster->controldata.ctrl_ver >= LARGE_OBJECT_SIZE_PG_CONTROL_VER) ||
!got_date_is_int || !got_float8_pass_by_value || !got_data_checksum_version)
{
pg_log(PG_REPORT,
"The %s cluster lacks some required control information:\n",
CLUSTER_NAME(cluster));
if (!got_xid)
pg_log(PG_REPORT, " checkpoint next XID\n");
if (!got_oid)
pg_log(PG_REPORT, " latest checkpoint next OID\n");
if (!got_multi)
pg_log(PG_REPORT, " latest checkpoint next MultiXactId\n");
if (!got_mxoff)
pg_log(PG_REPORT, " latest checkpoint next MultiXactOffset\n");
if (!got_oldestmulti &&
cluster->controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER)
pg_log(PG_REPORT, " latest checkpoint oldest MultiXactId\n");
if (!live_check && !got_nextxlogfile)
pg_log(PG_REPORT, " first WAL segment after reset\n");
if (!got_tli)
pg_log(PG_REPORT, " latest checkpoint timeline ID\n");
if (!got_align)
pg_log(PG_REPORT, " maximum alignment\n");
if (!got_blocksz)
pg_log(PG_REPORT, " block size\n");
if (!got_largesz)
pg_log(PG_REPORT, " large relation segment size\n");
if (!got_walsz)
pg_log(PG_REPORT, " WAL block size\n");
if (!got_walseg)
pg_log(PG_REPORT, " WAL segment size\n");
if (!got_ident)
pg_log(PG_REPORT, " maximum identifier length\n");
if (!got_index)
pg_log(PG_REPORT, " maximum number of indexed columns\n");
if (!got_toast)
pg_log(PG_REPORT, " maximum TOAST chunk size\n");
if (!got_large_object &&
cluster->controldata.ctrl_ver >= LARGE_OBJECT_SIZE_PG_CONTROL_VER)
pg_log(PG_REPORT, " large-object chunk size\n");
if (!got_date_is_int)
pg_log(PG_REPORT, " dates/times are integers?\n");
if (!got_float8_pass_by_value)
pg_log(PG_REPORT, " float8 argument passing method\n");
/* value added in Postgres 9.3 */
if (!got_data_checksum_version)
pg_log(PG_REPORT, " data checksum version\n");
pg_fatal("Cannot continue without required control information, terminating\n");
}
}
/*
* get_sock_dir
*
* Identify the socket directory to use for this cluster. If we're doing
* a live check (old cluster only), we need to find out where the postmaster
* is listening. Otherwise, we're going to put the socket into the current
* directory.
*/
void
get_sock_dir(ClusterInfo *cluster, bool live_check)
{
#ifdef HAVE_UNIX_SOCKETS
/*
* sockdir and port were added to postmaster.pid in PG 9.1. Pre-9.1 cannot
* process pg_ctl -w for sockets in non-default locations.
*/
if (GET_MAJOR_VERSION(cluster->major_version) >= 901)
{
if (!live_check)
{
/* Use the current directory for the socket */
cluster->sockdir = pg_malloc(MAXPGPATH);
if (!getcwd(cluster->sockdir, MAXPGPATH))
pg_fatal("cannot find current directory\n");
}
else
{
/*
* If we are doing a live check, we will use the old cluster's
* Unix domain socket directory so we can connect to the live
* server.
*/
unsigned short orig_port = cluster->port;
char filename[MAXPGPATH],
line[MAXPGPATH];
FILE *fp;
int lineno;
snprintf(filename, sizeof(filename), "%s/postmaster.pid",
cluster->pgdata);
if ((fp = fopen(filename, "r")) == NULL)
pg_fatal("Cannot open file %s: %m\n", filename);
for (lineno = 1;
lineno <= Max(LOCK_FILE_LINE_PORT, LOCK_FILE_LINE_SOCKET_DIR);
lineno++)
{
if (fgets(line, sizeof(line), fp) == NULL)
pg_fatal("Cannot read line %d from %s: %m\n", lineno, filename);
/* potentially overwrite user-supplied value */
if (lineno == LOCK_FILE_LINE_PORT)
sscanf(line, "%hu", &old_cluster.port);
if (lineno == LOCK_FILE_LINE_SOCKET_DIR)
{
cluster->sockdir = pg_strdup(line);
/* strip off newline */
if (strchr(cluster->sockdir, '\n') != NULL)
*strchr(cluster->sockdir, '\n') = '\0';
}
}
fclose(fp);
/* warn of port number correction */
if (orig_port != DEF_PGUPORT && old_cluster.port != orig_port)
pg_log(PG_WARNING, "User-supplied old port number %hu corrected to %hu\n",
orig_port, cluster->port);
}
}
else
/*
* Can't get sockdir and pg_ctl -w can't use a non-default, use
* default
*/
cluster->sockdir = NULL;
#else /* !HAVE_UNIX_SOCKETS */
cluster->sockdir = NULL;
#endif
}
/*
* gen_db_file_maps()
*
* generates database mappings for "old_db" and "new_db". Returns a malloc'ed
* array of mappings. nmaps is a return parameter which refers to the number
* mappings.
*/
FileNameMap *
gen_db_file_maps(DbInfo *old_db, DbInfo *new_db,
int *nmaps, const char *old_pgdata, const char *new_pgdata)
{
FileNameMap *maps;
int old_relnum, new_relnum;
int num_maps = 0;
maps = (FileNameMap *) pg_malloc(sizeof(FileNameMap) *
old_db->rel_arr.nrels);
/*
* The old database shouldn't have more relations than the new one.
* We force the new cluster to have a TOAST table if the old table
* had one.
*/
if (old_db->rel_arr.nrels > new_db->rel_arr.nrels)
pg_fatal("old and new databases \"%s\" have a mismatched number of relations\n",
old_db->db_name);
/* Drive the loop using new_relnum, which might be higher. */
for (old_relnum = new_relnum = 0; new_relnum < new_db->rel_arr.nrels;
new_relnum++)
{
RelInfo *old_rel;
RelInfo *new_rel = &new_db->rel_arr.rels[new_relnum];
/*
* It is possible that the new cluster has a TOAST table for a table
* that didn't need one in the old cluster, e.g. 9.0 to 9.1 changed the
* NUMERIC length computation. Therefore, if we have a TOAST table
* in the new cluster that doesn't match, skip over it and continue
* processing. It is possible this TOAST table used an OID that was
* reserved in the old cluster, but we have no way of testing that,
* and we would have already gotten an error at the new cluster schema
* creation stage. Fortunately, since we only restore the OID counter
* after schema restore, and restore in OID order via pg_dump, a
* conflict would only happen if the new TOAST table had a very low
* OID. However, TOAST tables created long after initial table
* creation can have any OID, particularly after OID wraparound.
*/
if (old_relnum == old_db->rel_arr.nrels)
{
if (strcmp(new_rel->nspname, "pg_toast") == 0)
continue;
else
pg_fatal("Extra non-TOAST relation found in database \"%s\": new OID %d\n",
old_db->db_name, new_rel->reloid);
}
old_rel = &old_db->rel_arr.rels[old_relnum];
if (old_rel->reloid != new_rel->reloid)
{
if (strcmp(new_rel->nspname, "pg_toast") == 0)
continue;
else
pg_fatal("Mismatch of relation OID in database \"%s\": old OID %d, new OID %d\n",
old_db->db_name, old_rel->reloid, new_rel->reloid);
}
/*
* TOAST table names initially match the heap pg_class oid. In
* pre-8.4, TOAST table names change during CLUSTER; in pre-9.0, TOAST
* table names change during ALTER TABLE ALTER COLUMN SET TYPE. In >=
* 9.0, TOAST relation names always use heap table oids, hence we
* cannot check relation names when upgrading from pre-9.0. Clusters
* upgraded to 9.0 will get matching TOAST names. If index names don't
* match primary key constraint names, this will fail because pg_dump
* dumps constraint names and pg_upgrade checks index names.
*/
if (strcmp(old_rel->nspname, new_rel->nspname) != 0 ||
((GET_MAJOR_VERSION(old_cluster.major_version) >= 900 ||
strcmp(old_rel->nspname, "pg_toast") != 0) &&
strcmp(old_rel->relname, new_rel->relname) != 0))
pg_fatal("Mismatch of relation names in database \"%s\": "
"old name \"%s.%s\", new name \"%s.%s\"\n",
old_db->db_name, old_rel->nspname, old_rel->relname,
new_rel->nspname, new_rel->relname);
create_rel_filename_map(old_pgdata, new_pgdata, old_db, new_db,
old_rel, new_rel, maps + num_maps);
num_maps++;
old_relnum++;
}
/* Did we fail to exhaust the old array? */
if (old_relnum != old_db->rel_arr.nrels)
pg_fatal("old and new databases \"%s\" have a mismatched number of relations\n",
old_db->db_name);
*nmaps = num_maps;
return maps;
}
/*
* Callback for processing target file list.
*
* All source files must be already processed before calling this. This only
* marks target data directory's files that didn't exist in the source for
* deletion.
*/
void
process_target_file(const char *path, file_type_t type, size_t oldsize,
const char *link_target)
{
bool exists;
char localpath[MAXPGPATH];
struct stat statbuf;
file_entry_t key;
file_entry_t *key_ptr;
filemap_t *map = filemap;
file_entry_t *entry;
snprintf(localpath, sizeof(localpath), "%s/%s", datadir_target, path);
if (lstat(localpath, &statbuf) < 0)
{
if (errno != ENOENT)
pg_fatal("could not stat file \"%s\": %s\n",
localpath, strerror(errno));
exists = false;
}
if (map->array == NULL)
{
/* on first call, initialize lookup array */
if (map->nlist == 0)
{
/* should not happen */
pg_fatal("source file list is empty\n");
}
filemap_list_to_array(map);
Assert(map->array != NULL);
qsort(map->array, map->narray, sizeof(file_entry_t *), path_cmp);
}
/*
* Completely ignore some special files
*/
if (strcmp(path, "postmaster.pid") == 0 ||
strcmp(path, "postmaster.opts") == 0)
return;
key.path = (char *) path;
key_ptr = &key;
exists = (bsearch(&key_ptr, map->array, map->narray, sizeof(file_entry_t *),
path_cmp) != NULL);
/* Remove any file or folder that doesn't exist in the source system. */
if (!exists)
{
entry = pg_malloc(sizeof(file_entry_t));
entry->path = pg_strdup(path);
entry->type = type;
entry->action = FILE_ACTION_REMOVE;
entry->oldsize = oldsize;
entry->newsize = 0;
entry->link_target = link_target ? pg_strdup(link_target) : NULL;
entry->next = NULL;
entry->pagemap.bitmap = NULL;
entry->pagemap.bitmapsize = 0;
entry->isrelfile = isRelDataFile(path);
if (map->last == NULL)
map->first = entry;
else
map->last->next = entry;
map->last = entry;
map->nlist++;
}
else
{
/*
* We already handled all files that exist in the source system in
* process_source_file().
*/
}
}
/*
* parallel_transfer_all_new_dbs
*
* This has the same API as transfer_all_new_dbs, except it does parallel execution
* by transfering multiple tablespaces in parallel
*/
void
parallel_transfer_all_new_dbs(DbInfoArr *old_db_arr, DbInfoArr *new_db_arr,
char *old_pgdata, char *new_pgdata,
char *old_tablespace)
{
#ifndef WIN32
pid_t child;
#else
HANDLE child;
transfer_thread_arg *new_arg;
#endif
if (user_opts.jobs <= 1)
/* throw_error must be true to allow jobs */
transfer_all_new_dbs(old_db_arr, new_db_arr, old_pgdata, new_pgdata, NULL);
else
{
/* parallel */
#ifdef WIN32
if (thread_handles == NULL)
thread_handles = pg_malloc(user_opts.jobs * sizeof(HANDLE));
if (transfer_thread_args == NULL)
{
int i;
transfer_thread_args = pg_malloc(user_opts.jobs * sizeof(transfer_thread_arg *));
/*
* For safety and performance, we keep the args allocated during
* the entire life of the process, and we don't free the args in a
* thread different from the one that allocated it.
*/
for (i = 0; i < user_opts.jobs; i++)
transfer_thread_args[i] = pg_malloc0(sizeof(transfer_thread_arg));
}
cur_thread_args = (void **) transfer_thread_args;
#endif
/* harvest any dead children */
while (reap_child(false) == true)
;
/* must we wait for a dead child? */
if (parallel_jobs >= user_opts.jobs)
reap_child(true);
/* set this before we start the job */
parallel_jobs++;
/* Ensure stdio state is quiesced before forking */
fflush(NULL);
#ifndef WIN32
child = fork();
if (child == 0)
{
transfer_all_new_dbs(old_db_arr, new_db_arr, old_pgdata, new_pgdata,
old_tablespace);
/* if we take another exit path, it will be non-zero */
/* use _exit to skip atexit() functions */
_exit(0);
}
else if (child < 0)
/* fork failed */
pg_fatal("could not create worker process: %s\n", strerror(errno));
#else
/* empty array element are always at the end */
new_arg = transfer_thread_args[parallel_jobs - 1];
/* Can only pass one pointer into the function, so use a struct */
new_arg->old_db_arr = old_db_arr;
new_arg->new_db_arr = new_db_arr;
if (new_arg->old_pgdata)
pg_free(new_arg->old_pgdata);
new_arg->old_pgdata = pg_strdup(old_pgdata);
if (new_arg->new_pgdata)
pg_free(new_arg->new_pgdata);
new_arg->new_pgdata = pg_strdup(new_pgdata);
if (new_arg->old_tablespace)
pg_free(new_arg->old_tablespace);
new_arg->old_tablespace = old_tablespace ? pg_strdup(old_tablespace) : NULL;
child = (HANDLE) _beginthreadex(NULL, 0, (void *) win32_transfer_all_new_dbs,
new_arg, 0, NULL);
if (child == 0)
pg_fatal("could not create worker thread: %s\n", strerror(errno));
thread_handles[parallel_jobs - 1] = child;
#endif
}
return;
}
/*
* This callback gets called while we read the WAL in the target, for every
* block that have changed in the target system. It makes note of all the
* changed blocks in the pagemap of the file.
*/
void
process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno)
{
char *path;
file_entry_t key;
file_entry_t *key_ptr;
file_entry_t *entry;
BlockNumber blkno_inseg;
int segno;
filemap_t *map = filemap;
file_entry_t **e;
Assert(map->array);
segno = blkno / RELSEG_SIZE;
blkno_inseg = blkno % RELSEG_SIZE;
path = datasegpath(rnode, forknum, segno);
key.path = (char *) path;
key_ptr = &key;
e = bsearch(&key_ptr, map->array, map->narray, sizeof(file_entry_t *),
path_cmp);
if (e)
entry = *e;
else
entry = NULL;
pfree(path);
if (entry)
{
Assert(entry->isrelfile);
switch (entry->action)
{
case FILE_ACTION_NONE:
case FILE_ACTION_TRUNCATE:
/* skip if we're truncating away the modified block anyway */
if ((blkno_inseg + 1) * BLCKSZ <= entry->newsize)
datapagemap_add(&entry->pagemap, blkno_inseg);
break;
case FILE_ACTION_COPY_TAIL:
/*
* skip the modified block if it is part of the "tail" that
* we're copying anyway.
*/
if ((blkno_inseg + 1) * BLCKSZ <= entry->oldsize)
datapagemap_add(&entry->pagemap, blkno_inseg);
break;
case FILE_ACTION_COPY:
case FILE_ACTION_REMOVE:
break;
case FILE_ACTION_CREATE:
pg_fatal("unexpected page modification for directory or symbolic link \"%s\"\n", entry->path);
}
}
else
{
/*
* If we don't have any record of this file in the file map, it means
* that it's a relation that doesn't exist in the source system, and
* it was subsequently removed in the target system, too. We can
* safely ignore it.
*/
}
}
/*
* parallel_exec_prog
*
* This has the same API as exec_prog, except it does parallel execution,
* and therefore must throw errors and doesn't return an error status.
*/
void
parallel_exec_prog(const char *log_file, const char *opt_log_file,
const char *fmt,...)
{
va_list args;
char cmd[MAX_STRING];
#ifndef WIN32
pid_t child;
#else
HANDLE child;
exec_thread_arg *new_arg;
#endif
va_start(args, fmt);
vsnprintf(cmd, sizeof(cmd), fmt, args);
va_end(args);
if (user_opts.jobs <= 1)
/* throw_error must be true to allow jobs */
exec_prog(log_file, opt_log_file, true, "%s", cmd);
else
{
/* parallel */
#ifdef WIN32
if (thread_handles == NULL)
thread_handles = pg_malloc(user_opts.jobs * sizeof(HANDLE));
if (exec_thread_args == NULL)
{
int i;
exec_thread_args = pg_malloc(user_opts.jobs * sizeof(exec_thread_arg *));
/*
* For safety and performance, we keep the args allocated during
* the entire life of the process, and we don't free the args in a
* thread different from the one that allocated it.
*/
for (i = 0; i < user_opts.jobs; i++)
exec_thread_args[i] = pg_malloc0(sizeof(exec_thread_arg));
}
cur_thread_args = (void **) exec_thread_args;
#endif
/* harvest any dead children */
while (reap_child(false) == true)
;
/* must we wait for a dead child? */
if (parallel_jobs >= user_opts.jobs)
reap_child(true);
/* set this before we start the job */
parallel_jobs++;
/* Ensure stdio state is quiesced before forking */
fflush(NULL);
#ifndef WIN32
child = fork();
if (child == 0)
/* use _exit to skip atexit() functions */
_exit(!exec_prog(log_file, opt_log_file, true, "%s", cmd));
else if (child < 0)
/* fork failed */
pg_fatal("could not create worker process: %s\n", strerror(errno));
#else
/* empty array element are always at the end */
new_arg = exec_thread_args[parallel_jobs - 1];
/* Can only pass one pointer into the function, so use a struct */
if (new_arg->log_file)
pg_free(new_arg->log_file);
new_arg->log_file = pg_strdup(log_file);
if (new_arg->opt_log_file)
pg_free(new_arg->opt_log_file);
new_arg->opt_log_file = opt_log_file ? pg_strdup(opt_log_file) : NULL;
if (new_arg->cmd)
pg_free(new_arg->cmd);
new_arg->cmd = pg_strdup(cmd);
child = (HANDLE) _beginthreadex(NULL, 0, (void *) win32_exec_prog,
new_arg, 0, NULL);
if (child == 0)
pg_fatal("could not create worker thread: %s\n", strerror(errno));
thread_handles[parallel_jobs - 1] = child;
#endif
}
return;
}
/*
* create_script_for_cluster_analyze()
*
* This incrementally generates better optimizer statistics
*/
void
create_script_for_cluster_analyze(char **analyze_script_file_name)
{
FILE *script = NULL;
char *user_specification = "";
prep_status("Creating script to analyze new cluster");
if (os_info.user_specified)
user_specification = psprintf("-U \"%s\" ", os_info.user);
*analyze_script_file_name = psprintf("analyze_new_cluster.%s", SCRIPT_EXT);
if ((script = fopen_priv(*analyze_script_file_name, "w")) == NULL)
pg_fatal("Could not open file \"%s\": %s\n",
*analyze_script_file_name, getErrorText(errno));
#ifndef WIN32
/* add shebang header */
fprintf(script, "#!/bin/sh\n\n");
#else
/* suppress command echoing */
fprintf(script, "@echo off\n");
#endif
fprintf(script, "echo %sThis script will generate minimal optimizer statistics rapidly%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %sso your system is usable, and then gather statistics twice more%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %swith increasing accuracy. When it is done, your system will%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %shave the default level of optimizer statistics.%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo%s\n\n", ECHO_BLANK);
fprintf(script, "echo %sIf you have used ALTER TABLE to modify the statistics target for%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %sany tables, you might want to remove them and restore them after%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %srunning this script because they will delay fast statistics generation.%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo%s\n\n", ECHO_BLANK);
fprintf(script, "echo %sIf you would like default statistics as quickly as possible, cancel%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %sthis script and run:%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %s \"%s/vacuumdb\" %s--all %s%s\n", ECHO_QUOTE,
new_cluster.bindir, user_specification,
/* Did we copy the free space files? */
(GET_MAJOR_VERSION(old_cluster.major_version) >= 804) ?
"--analyze-only" : "--analyze", ECHO_QUOTE);
fprintf(script, "echo%s\n\n", ECHO_BLANK);
#ifndef WIN32
fprintf(script, "sleep 2\n");
fprintf(script, "PGOPTIONS='-c default_statistics_target=1 -c vacuum_cost_delay=0'\n");
/* only need to export once */
fprintf(script, "export PGOPTIONS\n");
#else
fprintf(script, "REM simulate sleep 2\n");
fprintf(script, "PING 1.1.1.1 -n 1 -w 2000 > nul\n");
fprintf(script, "SET PGOPTIONS=-c default_statistics_target=1 -c vacuum_cost_delay=0\n");
#endif
fprintf(script, "echo %sGenerating minimal optimizer statistics (1 target)%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %s--------------------------------------------------%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "\"%s/vacuumdb\" %s--all --analyze-only\n",
new_cluster.bindir, user_specification);
fprintf(script, "echo%s\n", ECHO_BLANK);
fprintf(script, "echo %sThe server is now available with minimal optimizer statistics.%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %sQuery performance will be optimal once this script completes.%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo%s\n\n", ECHO_BLANK);
#ifndef WIN32
fprintf(script, "sleep 2\n");
fprintf(script, "PGOPTIONS='-c default_statistics_target=10'\n");
#else
fprintf(script, "REM simulate sleep\n");
fprintf(script, "PING 1.1.1.1 -n 1 -w 2000 > nul\n");
fprintf(script, "SET PGOPTIONS=-c default_statistics_target=10\n");
#endif
fprintf(script, "echo %sGenerating medium optimizer statistics (10 targets)%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %s---------------------------------------------------%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "\"%s/vacuumdb\" %s--all --analyze-only\n",
new_cluster.bindir, user_specification);
fprintf(script, "echo%s\n\n", ECHO_BLANK);
#ifndef WIN32
fprintf(script, "unset PGOPTIONS\n");
#else
fprintf(script, "SET PGOPTIONS\n");
#endif
fprintf(script, "echo %sGenerating default (full) optimizer statistics (100 targets?)%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "echo %s-------------------------------------------------------------%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fprintf(script, "\"%s/vacuumdb\" %s--all %s\n", new_cluster.bindir,
user_specification,
/* Did we copy the free space files? */
(GET_MAJOR_VERSION(old_cluster.major_version) >= 804) ?
"--analyze-only" : "--analyze");
fprintf(script, "echo%s\n\n", ECHO_BLANK);
fprintf(script, "echo %sDone%s\n",
ECHO_QUOTE, ECHO_QUOTE);
fclose(script);
#ifndef WIN32
if (chmod(*analyze_script_file_name, S_IRWXU) != 0)
pg_fatal("Could not add execute permission to file \"%s\": %s\n",
*analyze_script_file_name, getErrorText(errno));
#endif
if (os_info.user_specified)
pg_free(user_specification);
check_ok();
}
