Ejemplo n.º 1
5
StringInfo rest_call_with_lock(char *method, char *url, char *params, StringInfo postData, int64 mutex, bool shared, bool allowCancel) {
    CURL *curl;
    struct curl_slist *headers = NULL;
    char *errorbuff;

    StringInfo response = makeStringInfo();
    CURLcode ret;
    int64 response_code;

    errorbuff = (char *) palloc0(CURL_ERROR_SIZE);
    curl = curl_easy_init();

    if (curl) {
    	headers = curl_slist_append(headers, "Transfer-Encoding:");
	    headers = curl_slist_append(headers, "Expect:");

	    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
        curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 0L);   /* allow connections to be reused */
		if (allowCancel)
		{
			curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);      /* we want progress ... */
			curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, curl_progress_func);   /* to go here so we can detect a ^C within postgres */
		}
        curl_easy_setopt(curl, CURLOPT_USERAGENT, "zombodb for PostgreSQL");
        curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 0);
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_func);
        curl_easy_setopt(curl, CURLOPT_FAILONERROR, 0);
        curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorbuff);
        curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
        curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60 * 60L);  /* timeout of 60 minutes */

        curl_easy_setopt(curl, CURLOPT_URL, url);
        curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method);
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, response);
        curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, postData ? postData->len : 0);
        curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postData ? postData->data : NULL);
        curl_easy_setopt(curl, CURLOPT_POST, (strcmp(method, "POST") == 0) || (strcmp(method, "GET") != 0 && postData && postData->data) ? 1 : 0);
    } else {
        elog(IsTransactionState() ? ERROR : WARNING, "Unable to initialize libcurl");
    }

//	if (mutex != 0)
//	{
//		if (shared) DirectFunctionCall1(pg_advisory_lock_shared_int8, Int64GetDatum(mutex));
//		else DirectFunctionCall1(pg_advisory_lock_int8, Int64GetDatum(mutex));
//	}

    ret = curl_easy_perform(curl);

//	if (mutex != 0)
//	{
//		if (shared) DirectFunctionCall1(pg_advisory_unlock_shared_int8, Int64GetDatum(mutex));
//		else DirectFunctionCall1(pg_advisory_unlock_int8, Int64GetDatum(mutex));
//	}

    if (allowCancel && IsTransactionState() && InterruptPending) {
        /* we might have detected one in the progress function, so check for sure */
        CHECK_FOR_INTERRUPTS();
    }

    if (ret != 0) {
        /* curl messed up */
        elog(IsTransactionState() ? ERROR : WARNING, "libcurl error-code: %s(%d); message: %s; req=-X%s %s ", curl_easy_strerror(ret), ret, errorbuff, method, url);
    }

    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
    if (response_code < 200 || (response_code >=300 && response_code != 404)) {
        text *errorText = DatumGetTextP(DirectFunctionCall2(json_object_field_text, CStringGetTextDatum(response->data), CStringGetTextDatum("error")));

        elog(IsTransactionState() ? ERROR : WARNING, "rc=%ld; %s", response_code, errorText != NULL ? TextDatumGetCString(errorText) : response->data);
    }

    if (headers)
    	curl_slist_free_all(headers);
    curl_easy_cleanup(curl);
    pfree(errorbuff);

    return response;
}
Ejemplo n.º 2
0
/* check_hook: validate new default_tablespace */
bool
check_default_tablespace(char **newval, void **extra, GucSource source)
{
	/*
	 * If we aren't inside a transaction, we cannot do database access so
	 * cannot verify the name.  Must accept the value on faith.
	 */
	if (IsTransactionState())
	{
		if (**newval != '\0' &&
			!OidIsValid(get_tablespace_oid(*newval, true)))
		{
			/*
			 * When source == PGC_S_TEST, don't throw a hard error for a
			 * nonexistent tablespace, only a NOTICE.  See comments in guc.h.
			 */
			if (source == PGC_S_TEST)
			{
				ereport(NOTICE,
						(errcode(ERRCODE_UNDEFINED_OBJECT),
						 errmsg("tablespace \"%s\" does not exist",
								*newval)));
			}
			else
			{
				GUC_check_errdetail("Tablespace \"%s\" does not exist.",
									*newval);
				return false;
			}
		}
	}

	return true;
}
Ejemplo n.º 3
0
/*
 * Write logical decoding message into XLog.
 */
XLogRecPtr
LogLogicalMessage(const char *prefix, const char *message, size_t size,
				  bool transactional)
{
	xl_logical_message xlrec;

	/*
	 * Force xid to be allocated if we're emitting a transactional message.
	 */
	if (transactional)
	{
		Assert(IsTransactionState());
		GetCurrentTransactionId();
	}

	xlrec.dbId = MyDatabaseId;
	xlrec.transactional = transactional;
	xlrec.prefix_size = strlen(prefix) + 1;
	xlrec.message_size = size;

	XLogBeginInsert();
	XLogRegisterData((char *) &xlrec, SizeOfLogicalMessage);
	XLogRegisterData((char *) prefix, xlrec.prefix_size);
	XLogRegisterData((char *) message, size);

	/* allow origin filtering */
	XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);

	return XLogInsert(RM_LOGICALMSG_ID, XLOG_LOGICAL_MESSAGE);
}
Ejemplo n.º 4
0
finish_sync_worker(void)
{
	/*
	 * Commit any outstanding transaction. This is the usual case, unless
	 * there was nothing to do for the table.
	 */
	if (IsTransactionState())
	{
		CommitTransactionCommand();
		pgstat_report_stat(false);
	}

	/* And flush all writes. */
	XLogFlush(GetXLogWriteRecPtr());

	StartTransactionCommand();
	ereport(LOG,
			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
					MySubscription->name,
					get_rel_name(MyLogicalRepWorker->relid))));
	CommitTransactionCommand();

	/* Find the main apply worker and signal it. */
	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);

	/* Stop gracefully */
	proc_exit(0);
}
Ejemplo n.º 5
0
/*
 * Handle table synchronization cooperation from the synchronization
 * worker.
 *
 * If the sync worker is in CATCHUP state and reached (or passed) the
 * predetermined synchronization point in the WAL stream, mark the table as
 * SYNCDONE and finish.
 */
static void
process_syncing_tables_for_sync(XLogRecPtr current_lsn)
{
	Assert(IsTransactionState());

	SpinLockAcquire(&MyLogicalRepWorker->relmutex);

	if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP &&
		current_lsn >= MyLogicalRepWorker->relstate_lsn)
	{
		TimeLineID	tli;

		MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCDONE;
		MyLogicalRepWorker->relstate_lsn = current_lsn;

		SpinLockRelease(&MyLogicalRepWorker->relmutex);

		UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
								   MyLogicalRepWorker->relid,
								   MyLogicalRepWorker->relstate,
								   MyLogicalRepWorker->relstate_lsn);

		walrcv_endstreaming(wrconn, &tli);
		finish_sync_worker();
	}
	else
		SpinLockRelease(&MyLogicalRepWorker->relmutex);
}
Ejemplo n.º 6
0
/* assign_hook: validate new default_tablespace, do extra actions as needed */
const char *
assign_default_tablespace(const char *newval, bool doit, GucSource source)
{
	/*
	 * If we aren't inside a transaction, we cannot do database access so
	 * cannot verify the name.	Must accept the value on faith.
	 */
	if (IsTransactionState())
	{
		if (newval[0] != '\0' &&
			!OidIsValid(get_tablespace_oid(newval)))
		{
			/*
			 * When source == PGC_S_TEST, we are checking the argument of an
			 * ALTER DATABASE SET or ALTER USER SET command.  pg_dumpall dumps
			 * all roles before tablespaces, so if we're restoring a
			 * pg_dumpall script the tablespace might not yet exist, but will
			 * be created later.  Because of that, issue a NOTICE if source ==
			 * PGC_S_TEST, but accept the value anyway.
			 */
			ereport((source == PGC_S_TEST) ? NOTICE : GUC_complaint_elevel(source),
					(errcode(ERRCODE_UNDEFINED_OBJECT),
					 errmsg("tablespace \"%s\" does not exist",
							newval)));
			if (source == PGC_S_TEST)
				return newval;
			else
				return NULL;
		}
	}

	return newval;
}
Ejemplo n.º 7
0
/* GUC check_hook for default_text_search_config */
bool
check_TSCurrentConfig(char **newval, void **extra, GucSource source)
{
	/*
	 * If we aren't inside a transaction, we cannot do database access so
	 * cannot verify the config name.  Must accept it on faith.
	 */
	if (IsTransactionState())
	{
		Oid			cfgId;
		HeapTuple	tuple;
		Form_pg_ts_config cfg;
		char	   *buf;

		cfgId = get_ts_config_oid(stringToQualifiedNameList(*newval), true);

		/*
		 * When source == PGC_S_TEST, don't throw a hard error for a
		 * nonexistent configuration, only a NOTICE.  See comments in guc.h.
		 */
		if (!OidIsValid(cfgId))
		{
			if (source == PGC_S_TEST)
			{
				ereport(NOTICE,
						(errcode(ERRCODE_UNDEFINED_OBJECT),
						 errmsg("text search configuration \"%s\" does not exist", *newval)));
				return true;
			}
			else
				return false;
		}

		/*
		 * Modify the actually stored value to be fully qualified, to ensure
		 * later changes of search_path don't affect it.
		 */
		tuple = SearchSysCache1(TSCONFIGOID, ObjectIdGetDatum(cfgId));
		if (!HeapTupleIsValid(tuple))
			elog(ERROR, "cache lookup failed for text search configuration %u",
				 cfgId);
		cfg = (Form_pg_ts_config) GETSTRUCT(tuple);

		buf = quote_qualified_identifier(get_namespace_name(cfg->cfgnamespace),
										 NameStr(cfg->cfgname));

		ReleaseSysCache(tuple);

		/* GUC wants it malloc'd not palloc'd */
		free(*newval);
		*newval = strdup(buf);
		pfree(buf);
		if (!*newval)
			return false;
	}

	return true;
}
Ejemplo n.º 8
0
/*
 * Apply encoding conversion on src and return it. The encoding
 * conversion function is chosen from the pg_conversion system catalog
 * marked as "default". If it is not found in the schema search path,
 * it's taken from pg_catalog schema. If it even is not in the schema,
 * warn and returns src. We cannot raise an error, since it will cause
 * an infinit loop in error message sending.
 *
 * In the case of no conversion, src is returned.
 *
 * XXX We assume that storage for converted result is 4-to-1 growth in
 * the worst case. The rate for currently supported encoding pares are within 3
 * (SJIS JIS X0201 half width kanna -> UTF-8 is the worst case).
 * So "4" should be enough for the moment.
 */
unsigned char *
pg_do_encoding_conversion(unsigned char *src, int len,
						  int src_encoding, int dest_encoding)
{
	unsigned char *result;
	Oid			proc;

	if (!IsTransactionState())
		return src;

	if (src_encoding == dest_encoding)
		return src;

	if (src_encoding == PG_SQL_ASCII || dest_encoding == PG_SQL_ASCII)
		return src;

	if (len <= 0)
		return src;

	proc = FindDefaultConversionProc(src_encoding, dest_encoding);
	if (!OidIsValid(proc))
	{
		ereport(LOG,
				(errcode(ERRCODE_UNDEFINED_FUNCTION),
			errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist",
				   pg_encoding_to_char(src_encoding),
				   pg_encoding_to_char(dest_encoding))));
		return src;
	}

	/*
	 * XXX we should avoid throwing errors in OidFunctionCall. Otherwise
	 * we are going into infinite loop!  So we have to make sure that the
	 * function exists before calling OidFunctionCall.
	 */
	if (!SearchSysCacheExists(PROCOID,
							  ObjectIdGetDatum(proc),
							  0, 0, 0))
	{
		elog(LOG, "cache lookup failed for function %u", proc);
		return src;
	}

	result = palloc(len * 4 + 1);

	OidFunctionCall5(proc,
					 Int32GetDatum(src_encoding),
					 Int32GetDatum(dest_encoding),
					 CStringGetDatum(src),
					 CStringGetDatum(result),
					 Int32GetDatum(len));
	return result;
}
Ejemplo n.º 9
0
int rest_multi_call(MultiRestState *state, char *method, char *url, StringInfo postData, bool process) {
    int i;

    if (state->available == 0)
        rest_multi_partial_cleanup(state, false, false);

    if (state->available > 0) {
        for (i = 0; i < MAX_CURL_HANDLES; i++) {
            if (state->handles[i] == NULL) {
                CURL *curl;
                char *errorbuff;
                StringInfo response;
                int still_running;

                curl = state->handles[i] = curl_easy_init();
                if (!state->handles[i])
                    elog(IsTransactionState() ? ERROR : WARNING, "Unable to initialize CURL handle");

                errorbuff = state->errorbuffs[i] = palloc0(CURL_ERROR_SIZE);
                state->postDatas[i] = postData;
                response = state->responses[i] = makeStringInfo();

                curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1L);   /* elasticsearch tends to hang after reusing a connection around 8190 times */
                curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);      /* we want progress ... */
                curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, curl_progress_func);   /* ... to go here so we can detect a ^C within postgres */
                curl_easy_setopt(curl, CURLOPT_USERAGENT, "zombodb for PostgreSQL");
                curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 0);
                curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_func);
                curl_easy_setopt(curl, CURLOPT_FAILONERROR, 0);
                curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorbuff);
                curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
                curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60 * 60L);  /* timeout of 60 minutes */

                curl_easy_setopt(curl, CURLOPT_URL, url);
                curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method);
                curl_easy_setopt(curl, CURLOPT_WRITEDATA, response);
                curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, postData ? postData->len : 0);
                curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postData ? postData->data : NULL);
                curl_easy_setopt(curl, CURLOPT_POST, strcmp(method, "GET") != 0 && postData && postData->data ? 1 : 0);

                curl_multi_add_handle(state->multi_handle, curl);
                state->available--;

                if (process)
                    curl_multi_perform(state->multi_handle, &still_running);

                return i;
            }
        }
    }

    return -1;
}
Ejemplo n.º 10
0
/*
 * Handle ORIGIN message.
 *
 * TODO, support tracking of multiple origins
 */
static void
apply_handle_origin(StringInfo s)
{
	/*
	 * ORIGIN message can only come inside remote transaction and before any
	 * actual writes.
	 */
	if (!in_remote_transaction ||
		(IsTransactionState() && !am_tablesync_worker()))
		ereport(ERROR,
				(errcode(ERRCODE_PROTOCOL_VIOLATION),
				 errmsg("ORIGIN message sent out of order")));
}
Ejemplo n.º 11
0
/** used to check for Postgres-level interrupts. */
static int curl_progress_func(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
    /*
     * We only support detecting cancellation if we're actually in a transaction
     * i.e., we're not trying to COMMIT or ABORT a transaction
     */
    if (IsTransactionState()) {
        /*
         * This is what CHECK_FOR_INTERRUPTS() does,
         * except we want to gracefully exit out of
         * the libcurl innards before we let Postgres
         * throw the interrupt.
         */
        if (InterruptPending)
            return -1;
    }
    return 0;
}
Ejemplo n.º 12
0
/* check_hook: validate new default_tablespace */
bool
check_default_tablespace(char **newval, void **extra, GucSource source)
{
	/*
	 * If we aren't inside a transaction, we cannot do database access so
	 * cannot verify the name.	Must accept the value on faith.
	 */
	if (IsTransactionState())
	{
		if (**newval != '\0' &&
			!OidIsValid(get_tablespace_oid(*newval, true)))
		{
			GUC_check_errdetail("Tablespace \"%s\" does not exist.",
								*newval);
			return false;
		}
	}

	return true;
}
Ejemplo n.º 13
0
/*
 * Handle table synchronization cooperation from the apply worker.
 *
 * Walk over all subscription tables that are individually tracked by the
 * apply process (currently, all that have state other than
 * SUBREL_STATE_READY) and manage synchronization for them.
 *
 * If there are tables that need synchronizing and are not being synchronized
 * yet, start sync workers for them (if there are free slots for sync
 * workers).  To prevent starting the sync worker for the same relation at a
 * high frequency after a failure, we store its last start time with each sync
 * state info.  We start the sync worker for the same relation after waiting
 * at least wal_retrieve_retry_interval.
 *
 * For tables that are being synchronized already, check if sync workers
 * either need action from the apply worker or have finished.  This is the
 * SYNCWAIT to CATCHUP transition.
 *
 * If the synchronization position is reached (SYNCDONE), then the table can
 * be marked as READY and is no longer tracked.
 */
static void
process_syncing_tables_for_apply(XLogRecPtr current_lsn)
{
	struct tablesync_start_time_mapping
	{
		Oid			relid;
		TimestampTz last_start_time;
	};
	static List *table_states = NIL;
	static HTAB *last_start_times = NULL;
	ListCell   *lc;
	bool		started_tx = false;

	Assert(!IsTransactionState());

	/* We need up-to-date sync state info for subscription tables here. */
	if (!table_states_valid)
	{
		MemoryContext oldctx;
		List	   *rstates;
		ListCell   *lc;
		SubscriptionRelState *rstate;

		/* Clean the old list. */
		list_free_deep(table_states);
		table_states = NIL;

		StartTransactionCommand();
		started_tx = true;

		/* Fetch all non-ready tables. */
		rstates = GetSubscriptionNotReadyRelations(MySubscription->oid);

		/* Allocate the tracking info in a permanent memory context. */
		oldctx = MemoryContextSwitchTo(CacheMemoryContext);
		foreach(lc, rstates)
		{
			rstate = palloc(sizeof(SubscriptionRelState));
			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
			table_states = lappend(table_states, rstate);
		}
Ejemplo n.º 14
0
/*
 * Make sure that we started local transaction.
 *
 * Also switches to ApplyMessageContext as necessary.
 */
static bool
ensure_transaction(void)
{
	if (IsTransactionState())
	{
		SetCurrentStatementStartTimestamp();

		if (CurrentMemoryContext != ApplyMessageContext)
			MemoryContextSwitchTo(ApplyMessageContext);

		return false;
	}

	SetCurrentStatementStartTimestamp();
	StartTransactionCommand();

	maybe_reread_subscription();

	MemoryContextSwitchTo(ApplyMessageContext);
	return true;
}
Ejemplo n.º 15
0
/**
 * Get weight associated with queue. See queue.c.
 *
 * Attention is paid in order to avoid catalog lookups when not allowed.  The
 * superuser() function performs catalog lookups in certain cases. Also the
 * GetResqueueCapabilityEntry will always  do a catalog lookup. In such cases
 * use the default weight.
 */
static int
ResourceQueueGetPriorityWeight(Oid queueId)
{
	List	   *capabilitiesList = NULL;
	List	   *entry = NULL;
	ListCell   *le = NULL;
	int			weight = BackoffDefaultWeight();

	if (!IsTransactionState())
		return weight;

	if (superuser())
		return BackoffSuperuserStatementWeight();

	if (queueId == InvalidOid)
		return weight;

	capabilitiesList = GetResqueueCapabilityEntry(queueId);		/* This is a list of
																 * lists */

	if (!capabilitiesList)
		return weight;

	foreach(le, capabilitiesList)
	{
		Value	   *key = NULL;

		entry = (List *) lfirst(le);
		Assert(entry);
		key = (Value *) linitial(entry);
		Assert(key->type == T_Integer); /* This is resource type id */
		if (intVal(key) == PG_RESRCTYPE_PRIORITY)
		{
			Value	   *val = lsecond(entry);

			Assert(val->type == T_String);
			weight = BackoffPriorityValueToInt(strVal(val));
		}
	}
Ejemplo n.º 16
0
/* assign_hook: validate new default_tablespace, do extra actions as needed */
const char *
assign_default_tablespace(const char *newval, bool doit, GucSource source)
{
	/*
	 * If we aren't inside a transaction, we cannot do database access so
	 * cannot verify the name.	Must accept the value on faith.
	 */
	if (IsTransactionState())
	{
		if (newval[0] != '\0' &&
			!OidIsValid(get_tablespace_oid(newval)))
		{
			ereport(GUC_complaint_elevel(source),
					(errcode(ERRCODE_UNDEFINED_OBJECT),
					 errmsg("tablespace \"%s\" does not exist",
							newval)));
			return NULL;
		}
	}

	return newval;
}
Ejemplo n.º 17
0
/* check_hook: validate new default_tablespace */
bool
check_default_tablespace(char **newval, void **extra, GucSource source)
{
	/*
	 * If we aren't inside a transaction, we cannot do database access so
	 * cannot verify the name.	Must accept the value on faith.
	 */
	if (IsTransactionState())
	{
		if (**newval != '\0' &&
			!OidIsValid(get_tablespace_oid(*newval, true)))
		{
			/*
			 * When source == PGC_S_TEST, we are checking the argument of an
			 * ALTER DATABASE SET or ALTER USER SET command.  pg_dumpall dumps
			 * all roles before tablespaces, so if we're restoring a
			 * pg_dumpall script the tablespace might not yet exist, but will
			 * be created later.  Because of that, issue a NOTICE if source ==
			 * PGC_S_TEST, but accept the value anyway.
			 */
			if (source == PGC_S_TEST)
			{
				ereport(NOTICE,
						(errcode(ERRCODE_UNDEFINED_OBJECT),
						 errmsg("tablespace \"%s\" does not exist",
								*newval)));
			}
			else
			{
				GUC_check_errdetail("Tablespace \"%s\" does not exist.",
									*newval);
				return false;
			}
		}
	}

	return true;
}
Ejemplo n.º 18
0
/*
 * Handle COMMIT message.
 *
 * TODO, support tracking of multiple origins
 */
static void
apply_handle_commit(StringInfo s)
{
	LogicalRepCommitData commit_data;

	logicalrep_read_commit(s, &commit_data);

	Assert(commit_data.commit_lsn == remote_final_lsn);

	/* The synchronization worker runs in single transaction. */
	if (IsTransactionState() && !am_tablesync_worker())
	{
		/*
		 * Update origin state so we can restart streaming from correct
		 * position in case of crash.
		 */
		replorigin_session_origin_lsn = commit_data.end_lsn;
		replorigin_session_origin_timestamp = commit_data.committime;

		CommitTransactionCommand();
		pgstat_report_stat(false);

		store_flush_position(commit_data.end_lsn);
	}
	else
	{
		/* Process any invalidation messages that might have accumulated. */
		AcceptInvalidationMessages();
		maybe_reread_subscription();
	}

	in_remote_transaction = false;

	/* Process any tables that are being synchronized in parallel. */
	process_syncing_tables(commit_data.end_lsn);

	pgstat_report_activity(STATE_IDLE, NULL);
}
Ejemplo n.º 19
0
void rest_multi_partial_cleanup(MultiRestState *state, bool finalize, bool fast) {
    CURLMsg *msg;
    int msgs_left;

    while ((msg = curl_multi_info_read(state->multi_handle, &msgs_left))) {
        if (msg->msg == CURLMSG_DONE) {
            /** this handle is finished, so lets clean it */
            CURL *handle = msg->easy_handle;
            bool found = false;
            int i;

            curl_multi_remove_handle(state->multi_handle, handle);

            for (i=0; i<MAX_CURL_HANDLES; i++) {
                if (state->handles[i] == handle) {
                    long response_code = 0;

                    curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &response_code);
                    if (msg->data.result != 0 || response_code != 200 || strstr(state->responses[i]->data, "\"errors\":true")) {
                        /* REST endpoint messed up */
                        elog(IsTransactionState() ? ERROR : WARNING, "%s: %s", state->errorbuffs[i], state->responses[i]->data);
                    }

                    state->handles[i] = NULL;
                    state->available++;

                    if (state->errorbuffs[i] != NULL) {
                        pfree(state->errorbuffs[i]);
                        state->errorbuffs[i] = NULL;
                    }
                    if (state->postDatas[i] != NULL) {
                        pfree(state->postDatas[i]->data);
                        pfree(state->postDatas[i]);
                        state->postDatas[i] = NULL;
                    }
                    if (state->responses[i] != NULL) {
                        pfree(state->responses[i]->data);
                        pfree(state->responses[i]);
                        state->responses[i] = NULL;
                    }

                    curl_easy_cleanup(handle);

                    if (fast)
                        return;

                    found = true;
                    break;
                }
            }

            if (!found) {
                elog(IsTransactionState() ? ERROR : WARNING, "Couldn't find easy_handle for %p", handle);
            }
        }
    }

    if (finalize) {
        curl_multi_cleanup(state->multi_handle);

        state->available = MAX_CURL_HANDLES;
        state->multi_handle = NULL;
    }
}
Ejemplo n.º 20
0
/*
 * Set the client encoding and save fmgrinfo for the conversion
 * function if necessary.  Returns 0 if okay, -1 if not (bad encoding
 * or can't support conversion)
 */
int
SetClientEncoding(int encoding, bool doit)
{
	int			current_server_encoding;
	Oid			to_server_proc,
				to_client_proc;
	FmgrInfo   *to_server;
	FmgrInfo   *to_client;
	MemoryContext oldcontext;

	if (!PG_VALID_FE_ENCODING(encoding))
		return (-1);

	/* Can't do anything during startup, per notes above */
	if (!backend_startup_complete)
	{
		if (doit)
			pending_client_encoding = encoding;
		return 0;
	}

	current_server_encoding = GetDatabaseEncoding();

	/*
	 * Check for cases that require no conversion function.
	 */
	if (current_server_encoding == encoding ||
		(current_server_encoding == PG_SQL_ASCII ||
		 encoding == PG_SQL_ASCII))
	{
		if (doit)
		{
			ClientEncoding = &pg_enc2name_tbl[encoding];

			if (ToServerConvProc != NULL)
			{
				if (ToServerConvProc->fn_extra)
					pfree(ToServerConvProc->fn_extra);
				pfree(ToServerConvProc);
			}
			ToServerConvProc = NULL;

			if (ToClientConvProc != NULL)
			{
				if (ToClientConvProc->fn_extra)
					pfree(ToClientConvProc->fn_extra);
				pfree(ToClientConvProc);
			}
			ToClientConvProc = NULL;
		}
		return 0;
	}

	/*
	 * If we're not inside a transaction then we can't do catalog lookups,
	 * so fail.  After backend startup, this could only happen if we are
	 * re-reading postgresql.conf due to SIGHUP --- so basically this just
	 * constrains the ability to change client_encoding on the fly from
	 * postgresql.conf.  Which would probably be a stupid thing to do
	 * anyway.
	 */
	if (!IsTransactionState())
		return -1;

	/*
	 * Look up the conversion functions.
	 */
	to_server_proc = FindDefaultConversionProc(encoding,
											   current_server_encoding);
	if (!OidIsValid(to_server_proc))
		return -1;
	to_client_proc = FindDefaultConversionProc(current_server_encoding,
											   encoding);
	if (!OidIsValid(to_client_proc))
		return -1;

	/*
	 * Done if not wanting to actually apply setting.
	 */
	if (!doit)
		return 0;

	/*
	 * load the fmgr info into TopMemoryContext so that it survives
	 * outside transaction.
	 */
	oldcontext = MemoryContextSwitchTo(TopMemoryContext);
	to_server = palloc(sizeof(FmgrInfo));
	to_client = palloc(sizeof(FmgrInfo));
	fmgr_info(to_server_proc, to_server);
	fmgr_info(to_client_proc, to_client);
	MemoryContextSwitchTo(oldcontext);

	ClientEncoding = &pg_enc2name_tbl[encoding];

	if (ToServerConvProc != NULL)
	{
		if (ToServerConvProc->fn_extra)
			pfree(ToServerConvProc->fn_extra);
		pfree(ToServerConvProc);
	}
	ToServerConvProc = to_server;

	if (ToClientConvProc != NULL)
	{
		if (ToClientConvProc->fn_extra)
			pfree(ToClientConvProc->fn_extra);
		pfree(ToClientConvProc);
	}
	ToClientConvProc = to_client;

	return 0;
}
Ejemplo n.º 21
0
bool pljavaViableXact()
{
	return IsTransactionState() && 'E' != TransactionBlockStatusCode();
}
Ejemplo n.º 22
0
/*
 * ContinuousQueryWorkerStartup
 *
 * Launches a CQ worker, which continuously generates partial query results to send
 * back to the combiner process.
 */
void
ContinuousQueryWorkerRun(Portal portal, ContinuousViewState *state, QueryDesc *queryDesc, ResourceOwner owner)
{
	EState	   *estate = NULL;
	DestReceiver *dest;
	CmdType		operation;
	MemoryContext oldcontext;
	int timeoutms = state->maxwaitms;
	MemoryContext runcontext;
	CQProcEntry *entry = GetCQProcEntry(MyCQId);
	ResourceOwner cqowner = ResourceOwnerCreate(NULL, "CQResourceOwner");
	bool savereadonly = XactReadOnly;

	cq_stat_initialize(state->viewid, MyProcPid);

	dest = CreateDestReceiver(DestCombiner);
	SetCombinerDestReceiverParams(dest, MyCQId);

	/* workers only need read-only transactions */
	XactReadOnly = true;

	runcontext = AllocSetContextCreate(TopMemoryContext, "CQRunContext",
			ALLOCSET_DEFAULT_MINSIZE,
			ALLOCSET_DEFAULT_INITSIZE,
			ALLOCSET_DEFAULT_MAXSIZE);

	elog(LOG, "\"%s\" worker %d running", queryDesc->plannedstmt->cq_target->relname, MyProcPid);
	MarkWorkerAsRunning(MyCQId, MyWorkerId);
	pgstat_report_activity(STATE_RUNNING, queryDesc->sourceText);

	TupleBufferInitLatch(WorkerTupleBuffer, MyCQId, MyWorkerId, &MyProc->procLatch);

	oldcontext = MemoryContextSwitchTo(runcontext);

retry:
	PG_TRY();
	{
		bool xact_commit = true;
		TimestampTz last_process = GetCurrentTimestamp();
		TimestampTz last_commit = GetCurrentTimestamp();

		start_executor(queryDesc, runcontext, cqowner);

		CurrentResourceOwner = cqowner;

		estate = queryDesc->estate;
		operation = queryDesc->operation;

		/*
		 * Initialize context that lives for the duration of a single iteration
		 * of the main worker loop
		 */
		CQExecutionContext = AllocSetContextCreate(estate->es_query_cxt, "CQExecutionContext",
				ALLOCSET_DEFAULT_MINSIZE,
				ALLOCSET_DEFAULT_INITSIZE,
				ALLOCSET_DEFAULT_MAXSIZE);

		estate->es_lastoid = InvalidOid;

		/*
		 * Startup combiner receiver
		 */
		(*dest->rStartup) (dest, operation, queryDesc->tupDesc);

		for (;;)
		{
			if (!TupleBufferHasUnreadSlots())
			{
				if (TimestampDifferenceExceeds(last_process, GetCurrentTimestamp(), state->emptysleepms))
				{
					/* force stats flush */
					cq_stat_report(true);

					pgstat_report_activity(STATE_IDLE, queryDesc->sourceText);
					TupleBufferWait(WorkerTupleBuffer, MyCQId, MyWorkerId);
					pgstat_report_activity(STATE_RUNNING, queryDesc->sourceText);
				}
				else
					pg_usleep(Min(WAIT_SLEEP_MS, state->emptysleepms) * 1000);
			}

			TupleBufferResetNotify(WorkerTupleBuffer, MyCQId, MyWorkerId);

			if (xact_commit)
				StartTransactionCommand();

			set_snapshot(estate, cqowner);
			CurrentResourceOwner = cqowner;
			MemoryContextSwitchTo(estate->es_query_cxt);

			estate->es_processed = 0;
			estate->es_filtered = 0;

			/*
			 * Run plan on a microbatch
			 */
			ExecutePlan(estate, queryDesc->planstate, operation,
					true, 0, timeoutms, ForwardScanDirection, dest);

			IncrementCQExecutions(1);
			TupleBufferClearPinnedSlots();

			if (state->long_xact)
			{
				if (TimestampDifferenceExceeds(last_commit, GetCurrentTimestamp(), LONG_RUNNING_XACT_DURATION))
					xact_commit = true;
				else
					xact_commit = false;
			}

			unset_snapshot(estate, cqowner);
			if (xact_commit)
			{
				CommitTransactionCommand();
				last_commit = GetCurrentTimestamp();
			}

			MemoryContextResetAndDeleteChildren(CQExecutionContext);
			MemoryContextSwitchTo(runcontext);
			CurrentResourceOwner = cqowner;

			if (estate->es_processed || estate->es_filtered)
			{
				/*
				 * If the CV query is such that the select does not return any tuples
				 * ex: select id where id=99; and id=99 does not exist, then this reset
				 * will fail. What will happen is that the worker will block at the latch for every
				 * allocated slot, TILL a cv returns a non-zero tuple, at which point
				 * the worker will resume a simple sleep for the threshold time.
				 */
				last_process = GetCurrentTimestamp();

				/*
				 * Send stats to the collector
				 */
				cq_stat_report(false);
			}

			/* Has the CQ been deactivated? */
			if (!entry->active)
			{
				if (ActiveSnapshotSet())
					unset_snapshot(estate, cqowner);
				if (IsTransactionState())
					CommitTransactionCommand();
				break;
			}
		}

		CurrentResourceOwner = cqowner;

		/*
		 * The cleanup functions below expect these things to be registered
		 */
		RegisterSnapshotOnOwner(estate->es_snapshot, cqowner);
		RegisterSnapshotOnOwner(queryDesc->snapshot, cqowner);
		RegisterSnapshotOnOwner(queryDesc->crosscheck_snapshot, cqowner);

		/* cleanup */
		ExecutorFinish(queryDesc);
		ExecutorEnd(queryDesc);
		FreeQueryDesc(queryDesc);
	}
	PG_CATCH();
	{
		EmitErrorReport();
		FlushErrorState();

		/* Since the worker is read-only, we can simply commit the transaction. */
		if (ActiveSnapshotSet())
			unset_snapshot(estate, cqowner);
		if (IsTransactionState())
			CommitTransactionCommand();

		TupleBufferUnpinAllPinnedSlots();
		TupleBufferClearReaders();

		/* This resets the es_query_ctx and in turn the CQExecutionContext */
		MemoryContextResetAndDeleteChildren(runcontext);

		IncrementCQErrors(1);

		if (continuous_query_crash_recovery)
			goto retry;
	}
	PG_END_TRY();

	(*dest->rShutdown) (dest);

	MemoryContextSwitchTo(oldcontext);
	MemoryContextDelete(runcontext);

	XactReadOnly = savereadonly;

	/*
	 * Remove proc-level stats
	 */
	cq_stat_report(true);
	cq_stat_send_purge(state->viewid, MyProcPid, CQ_STAT_WORKER);

	CurrentResourceOwner = owner;
}
Ejemplo n.º 23
0
/*
 * Set the client encoding and save fmgrinfo for the conversion
 * function if necessary.  Returns 0 if okay, -1 if not (bad encoding
 * or can't support conversion)
 */
int
SetClientEncoding(int encoding, bool doit)
{
	int			current_server_encoding;
	Oid			to_server_proc,
				to_client_proc;
	FmgrInfo   *to_server;
	FmgrInfo   *to_client;
	MemoryContext oldcontext;

	if (!PG_VALID_FE_ENCODING(encoding))
		return -1;

	/* Can't do anything during startup, per notes above */
	if (!backend_startup_complete)
	{
		if (doit)
			pending_client_encoding = encoding;
		return 0;
	}

	current_server_encoding = GetDatabaseEncoding();

	/*
	 * Check for cases that require no conversion function.
	 */
	if (current_server_encoding == encoding ||
		current_server_encoding == PG_SQL_ASCII ||
		encoding == PG_SQL_ASCII)
	{
		if (doit)
		{
			ClientEncoding = &pg_enc2name_tbl[encoding];
			ToServerConvProc = NULL;
			ToClientConvProc = NULL;
			if (MbProcContext)
				MemoryContextReset(MbProcContext);
		}
		return 0;
	}

	/*
	 * If we're not inside a transaction then we can't do catalog lookups, so
	 * fail.  After backend startup, this could only happen if we are
	 * re-reading postgresql.conf due to SIGHUP --- so basically this just
	 * constrains the ability to change client_encoding on the fly from
	 * postgresql.conf.  Which would probably be a stupid thing to do anyway.
	 */
	if (!IsTransactionState())
		return -1;

	/*
	 * Look up the conversion functions.
	 */
	to_server_proc = FindDefaultConversionProc(encoding,
											   current_server_encoding);
	if (!OidIsValid(to_server_proc))
		return -1;
	to_client_proc = FindDefaultConversionProc(current_server_encoding,
											   encoding);
	if (!OidIsValid(to_client_proc))
		return -1;

	/*
	 * Done if not wanting to actually apply setting.
	 */
	if (!doit)
		return 0;

	/* Before loading the new fmgr info, remove the old info, if any */
	ToServerConvProc = NULL;
	ToClientConvProc = NULL;
	if (MbProcContext != NULL)
	{
		MemoryContextReset(MbProcContext);
	}
	else
	{
		/*
		 * This is the first time through, so create the context. Make it a
		 * child of TopMemoryContext so that these values survive across
		 * transactions.
		 */
		MbProcContext = AllocSetContextCreate(TopMemoryContext,
											  "MbProcContext",
											  ALLOCSET_SMALL_MINSIZE,
											  ALLOCSET_SMALL_INITSIZE,
											  ALLOCSET_SMALL_MAXSIZE);
	}

	/* Load the fmgr info into MbProcContext */
	oldcontext = MemoryContextSwitchTo(MbProcContext);
	to_server = palloc(sizeof(FmgrInfo));
	to_client = palloc(sizeof(FmgrInfo));
	fmgr_info(to_server_proc, to_server);
	fmgr_info(to_client_proc, to_client);
	MemoryContextSwitchTo(oldcontext);

	ClientEncoding = &pg_enc2name_tbl[encoding];
	ToServerConvProc = to_server;
	ToClientConvProc = to_client;

	return 0;
}
Ejemplo n.º 24
0
/*
 * Apply encoding conversion on src and return it. The encoding
 * conversion function is chosen from the pg_conversion system catalog
 * marked as "default". If it is not found in the schema search path,
 * it's taken from pg_catalog schema. If it even is not in the schema,
 * warn and return src.
 *
 * If conversion occurs, a palloc'd null-terminated string is returned.
 * In the case of no conversion, src is returned.
 *
 * CAUTION: although the presence of a length argument means that callers
 * can pass non-null-terminated strings, care is required because the same
 * string will be passed back if no conversion occurs.	Such callers *must*
 * check whether result == src and handle that case differently.
 *
 * Note: we try to avoid raising error, since that could get us into
 * infinite recursion when this function is invoked during error message
 * sending.  It should be OK to raise error for overlength strings though,
 * since the recursion will come with a shorter message.
 */
unsigned char *
pg_do_encoding_conversion(unsigned char *src, int len,
						  int src_encoding, int dest_encoding)
{
	unsigned char *result;
	Oid			proc;

	if (!IsTransactionState())
		return src;

	if (src_encoding == dest_encoding)
		return src;

	if (src_encoding == PG_SQL_ASCII || dest_encoding == PG_SQL_ASCII)
		return src;

	if (len <= 0)
		return src;

	proc = FindDefaultConversionProc(src_encoding, dest_encoding);
	if (!OidIsValid(proc))
	{
		ereport(LOG,
				(errcode(ERRCODE_UNDEFINED_FUNCTION),
				 errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist",
						pg_encoding_to_char(src_encoding),
						pg_encoding_to_char(dest_encoding))));
		return src;
	}

	/*
	 * XXX we should avoid throwing errors in OidFunctionCall. Otherwise we
	 * are going into infinite loop!  So we have to make sure that the
	 * function exists before calling OidFunctionCall.
	 */
	/* XXX: would have been function_exists() */
	if (!(caql_getcount(
					NULL,
					cql("SELECT COUNT(*) FROM pg_proc "
						" WHERE oid = :1 ",
						ObjectIdGetDatum(proc))) > 0))
	{
		elog(LOG, "cache lookup failed for function %u", proc);
		return src;
	}

	/*
	 * Allocate space for conversion result, being wary of integer overflow
	 */
	if ((Size) len >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH))
		ereport(ERROR,
				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
				 errmsg("out of memory"),
		 errdetail("String of %d bytes is too long for encoding conversion.",
				   len)));

	result = palloc(len * MAX_CONVERSION_GROWTH + 1);

	OidFunctionCall5(proc,
					 Int32GetDatum(src_encoding),
					 Int32GetDatum(dest_encoding),
					 CStringGetDatum((char *)src),
					 CStringGetDatum((char *)result),
					 Int32GetDatum(len));
	return result;
}
Ejemplo n.º 25
0
/*
 * PrepareTempTablespaces -- prepare to use temp tablespaces
 *
 * If we have not already done so in the current transaction, parse the
 * temp_tablespaces GUC variable and tell fd.c which tablespace(s) to use
 * for temp files.
 */
void
PrepareTempTablespaces(void)
{
	char	   *rawname;
	List	   *namelist;
	Oid		   *tblSpcs;
	int			numSpcs;
	ListCell   *l;

	/* No work if already done in current transaction */
	if (TempTablespacesAreSet())
		return;

	/*
	 * Can't do catalog access unless within a transaction.  This is just a
	 * safety check in case this function is called by low-level code that
	 * could conceivably execute outside a transaction.  Note that in such a
	 * scenario, fd.c will fall back to using the current database's default
	 * tablespace, which should always be OK.
	 */
	if (!IsTransactionState())
		return;

	/* Need a modifiable copy of string */
	rawname = pstrdup(temp_tablespaces);

	/* Parse string into list of identifiers */
	if (!SplitIdentifierString(rawname, ',', &namelist))
	{
		/* syntax error in name list */
		SetTempTablespaces(NULL, 0);
		pfree(rawname);
		list_free(namelist);
		return;
	}

	/* Store tablespace OIDs in an array in TopTransactionContext */
	tblSpcs = (Oid *) MemoryContextAlloc(TopTransactionContext,
										 list_length(namelist) * sizeof(Oid));
	numSpcs = 0;
	foreach(l, namelist)
	{
		char	   *curname = (char *) lfirst(l);
		Oid			curoid;
		AclResult	aclresult;

		/* Allow an empty string (signifying database default) */
		if (curname[0] == '\0')
		{
			tblSpcs[numSpcs++] = InvalidOid;
			continue;
		}

		/* Else verify that name is a valid tablespace name */
		curoid = get_tablespace_oid(curname);
		if (curoid == InvalidOid)
		{
			/* Silently ignore any bad list elements */
			continue;
		}

		/*
		 * Allow explicit specification of database's default tablespace in
		 * temp_tablespaces without triggering permissions checks.
		 */
		if (curoid == MyDatabaseTableSpace)
		{
			tblSpcs[numSpcs++] = InvalidOid;
			continue;
		}

		/* Check permissions similarly */
		aclresult = pg_tablespace_aclcheck(curoid, GetUserId(),
										   ACL_CREATE);
		if (aclresult != ACLCHECK_OK)
			continue;

		tblSpcs[numSpcs++] = curoid;
	}