/*
* Write a file range to a temporary table in the server.
*
* The range is sent to the server as a COPY formatted line, to be inserted
* into the 'fetchchunks' temporary table. It is used in receiveFileChunks()
* function to actually fetch the data.
*/
static void
fetch_file_range(const char *path, uint64 begin, uint64 end)
{
char linebuf[MAXPGPATH + 23];
/* Split the range into CHUNKSIZE chunks */
while (end - begin > 0)
{
unsigned int len;
/* Fine as long as CHUNKSIZE is not bigger than UINT32_MAX */
if (end - begin > CHUNKSIZE)
len = CHUNKSIZE;
else
len = (unsigned int) (end - begin);
snprintf(linebuf, sizeof(linebuf), "%s\t" UINT64_FORMAT "\t%u\n", path, begin, len);
if (PQputCopyData(conn, linebuf, strlen(linebuf)) != 1)
pg_fatal("could not send COPY data: %s",
PQerrorMessage(conn));
begin += len;
}
}
s_object *
RS_PostgreSQL_CopyIn(Con_Handle * conHandle, s_object * filename)
{
S_EVALUATOR RS_DBI_connection * con;
PGconn *my_connection;
char *dyn_filename;
char copybuf[COPY_IN_BUFSIZE];
FILE* filehandle;
size_t len;
int pqretcode;
con = RS_DBI_getConnection(conHandle);
my_connection = (PGconn *) con->drvConnection;
dyn_filename = RS_DBI_copyString(CHR_EL(filename, 0));
filehandle=fopen(dyn_filename, "r");
if(filehandle == NULL){
char errmsg[1024];
snprintf(errmsg, 1024, "could not open file: %s", dyn_filename);
RS_DBI_errorMessage(dyn_filename, RS_DBI_ERROR);
return S_NULL_ENTRY;
}
while((len = fread(copybuf,1,COPY_IN_BUFSIZE, filehandle))){
pqretcode = PQputCopyData(my_connection, copybuf, len);
chkpqcopydataerr(my_connection, pqretcode);
}
PQputCopyEnd(my_connection, NULL);
fclose(filehandle);
free(dyn_filename);
return S_NULL_ENTRY;
}
static void
copy_file_range(const char *path, unsigned int begin, unsigned int end)
{
char linebuf[MAXPGPATH + 23];
/* Split the range into CHUNKSIZE chunks */
while (end - begin > 0)
{
unsigned int len;
if (end - begin > CHUNKSIZE)
len = CHUNKSIZE;
else
len = end - begin;
snprintf(linebuf, sizeof(linebuf), "%s\t%u\t%u\n", path, begin, len);
if (PQputCopyData(conn, linebuf, strlen(linebuf)) != 1)
{
fprintf(stderr, "error sending COPY data: %s\n",
PQerrorMessage(conn));
exit(1);
}
begin += len;
}
}
bool pgConn::PutCopyData(const char *data, long count)
{
// Execute the query and get the status
int result = PQputCopyData(conn, data, count);
return result == 1;
}
int PG_cache_dbop_copy(struct DBdesc *db, struct db_cache *cache_elem, struct insert_data *idata)
{
PGresult *ret;
char *ptr_values, *ptr_where;
char default_delim[] = ",", delim_buf[SRVBUFLEN];
int num=0, have_flows=0;
if (config.what_to_count & COUNT_FLOWS) have_flows = TRUE;
if (!config.sql_delimiter)
snprintf(delim_buf, SRVBUFLEN, "%s", default_delim);
else
snprintf(delim_buf, SRVBUFLEN, "%s", config.sql_delimiter);
/* constructing SQL query */
ptr_where = where_clause;
ptr_values = values_clause;
memset(where_clause, 0, sizeof(where_clause));
memset(values_clause, 0, sizeof(values_clause));
memcpy(&values, ©_values, sizeof(values));
while (num < idata->num_primitives) {
(*where[num].handler)(cache_elem, idata, num, &ptr_values, &ptr_where);
num++;
}
#if defined HAVE_64BIT_COUNTERS
if (have_flows) snprintf(ptr_values, SPACELEFT(values_clause), "%s%llu%s%llu%s%llu\n", delim_buf, cache_elem->packet_counter,
delim_buf, cache_elem->bytes_counter,
delim_buf, cache_elem->flows_counter);
else snprintf(ptr_values, SPACELEFT(values_clause), "%s%llu%s%llu\n", delim_buf, cache_elem->packet_counter,
delim_buf, cache_elem->bytes_counter);
#else
if (have_flows) snprintf(ptr_values, SPACELEFT(values_clause), "%s%lu%s%lu%s%lu\n", delim_buf, cache_elem->packet_counter,
delim_buf, cache_elem->bytes_counter,
delim_buf, cache_elem->flows_counter);
else snprintf(ptr_values, SPACELEFT(values_clause), "%s%lu%s%lu\n", delim_buf, cache_elem->packet_counter,
delim_buf, cache_elem->bytes_counter);
#endif
strncpy(sql_data, values_clause, SPACELEFT(sql_data));
if (PQputCopyData(db->desc, sql_data, strlen(sql_data)) < 0) { // avoid strlen()
db->errmsg = PQerrorMessage(db->desc);
Log(LOG_DEBUG, "DEBUG ( %s/%s ): FAILED query follows:\n%s\n", config.name, config.type, sql_data);
if (db->errmsg) Log(LOG_ERR, "ERROR ( %s/%s ): %s\n", config.name, config.type, db->errmsg);
sql_db_fail(db);
return TRUE;
}
idata->iqn++;
idata->een++;
Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n", config.name, config.type, sql_data);
return FALSE;
}
/*
* Send a message to XLOG stream.
*
* ereports on error.
*/
static void
libpqrcv_send(const char *buffer, int nbytes)
{
if (PQputCopyData(streamConn, buffer, nbytes) <= 0 ||
PQflush(streamConn))
ereport(ERROR,
(errmsg("could not send data to WAL stream: %s",
PQerrorMessage(streamConn))));
}
static HB_BOOL addToContext( pgCopyContext * context, const char c )
{
if( context->position == context->length )
{
if( PQputCopyData( context->connection, context->buffer, context->position ) == -1 )
return HB_FALSE;
context->position = 0;
}
context->buffer[ context->position++ ] = ( HB_BYTE ) c;
return HB_TRUE;
}
/*
* Implement ahwrite() for direct-to-DB restore
*/
int
ExecuteSqlCommandBuf(Archive *AHX, const char *buf, size_t bufLen)
{
ArchiveHandle *AH = (ArchiveHandle *) AHX;
if (AH->outputKind == OUTPUT_COPYDATA)
{
/*
* COPY data.
*
* We drop the data on the floor if libpq has failed to enter COPY
* mode; this allows us to behave reasonably when trying to continue
* after an error in a COPY command.
*/
if (AH->pgCopyIn &&
PQputCopyData(AH->connection, buf, bufLen) <= 0)
exit_horribly(modulename, "error returned by PQputCopyData: %s",
PQerrorMessage(AH->connection));
}
else if (AH->outputKind == OUTPUT_OTHERDATA)
{
/*
* Table data expressed as INSERT commands; or, in old dump files,
* BLOB COMMENTS data (which is expressed as COMMENT ON commands).
*/
ExecuteSimpleCommands(AH, buf, bufLen);
}
else
{
/*
* General SQL commands; we assume that commands will not be split
* across calls.
*
* In most cases the data passed to us will be a null-terminated
* string, but if it's not, we have to add a trailing null.
*/
if (buf[bufLen] == '\0')
ExecuteSqlCommand(AH, buf, "could not execute query");
else
{
char *str = (char *) pg_malloc(bufLen + 1);
memcpy(str, buf, bufLen);
str[bufLen] = '\0';
ExecuteSqlCommand(AH, str, "could not execute query");
free(str);
}
}
return bufLen;
}
static HB_BOOL addStrnToContext( pgCopyContext * context, const char * str, HB_SIZE size )
{
HB_SIZE nSize = 0;
while( nSize < size )
{
if( context->connection == NULL || context->position == context->length )
{
if( PQputCopyData( context->connection, context->buffer, context->position ) == -1 )
return HB_FALSE;
context->position = 0;
}
context->buffer[ context->position++ ] = ( HB_BYTE ) str[ nSize++ ];
}
return HB_TRUE;
}
/*
* call-seq:
* conn.putline( str) -> nil
* conn.putline( ary) -> nil
* conn.putline( str) { ... } -> nil
*
* Sends the string to the backend server.
* You have to open the stream with a +COPY+ command using +copy_stdin+.
*
* If +str+ doesn't end in a newline, one is appended. If the argument
* is +ary+, a line will be built using +stringize_line+.
*
* If the connection is in nonblocking mode and no data could be sent
* the closure will be called and its value will be returned.
*/
VALUE
pgconn_putline( VALUE self, VALUE arg)
{
struct pgconn_data *c;
VALUE str;
const char *p;
int l;
int r;
switch (TYPE( arg)) {
case T_STRING:
str = arg;
break;
case T_ARRAY:
str = pgconn_stringize_line( self, arg);
break;
default:
str = rb_obj_as_string( arg);
break;
}
if (RSTRING_PTR( str)[ RSTRING_LEN( str) - 1] != '\n') {
VALUE t;
t = rb_str_dup( str);
rb_str_buf_cat( t, "\n", 1);
str = t;
}
Data_Get_Struct( self, struct pgconn_data, c);
p = pgconn_destring( c, str, &l);
r = PQputCopyData( c->conn, p, l);
if (r < 0)
rb_raise( rb_ePgConnCopy, "Copy from stdin failed.");
else if (r == 0)
return rb_yield( Qnil);
return Qnil;
}
/*
* Write a file range to a temporary table in the server.
*
* The range is sent to the server as a COPY formatted line, to be inserted
* into the 'fetchchunks' temporary table. It is used in receiveFileChunks()
* function to actually fetch the data.
*/
static void
fetch_file_range(const char *path, unsigned int begin, unsigned int end)
{
char linebuf[MAXPGPATH + 23];
/* Split the range into CHUNKSIZE chunks */
while (end - begin > 0)
{
unsigned int len;
if (end - begin > CHUNKSIZE)
len = CHUNKSIZE;
else
len = end - begin;
snprintf(linebuf, sizeof(linebuf), "%s\t%u\t%u\n", path, begin, len);
if (PQputCopyData(conn, linebuf, strlen(linebuf)) != 1)
pg_fatal("could not send COPY data: %s",
PQerrorMessage(conn));
begin += len;
}
}
static PGresult *
RemoteLoad(PGconn *conn, FILE *copystream, bool isbinary)
{
bool OK;
char buf[COPYBUFSIZ];
OK = true;
if (isbinary)
{
while (!interrupted)
{
int buflen;
buflen = fread(buf, 1, COPYBUFSIZ, copystream);
if (buflen <= 0)
break;
if (PQputCopyData(conn, buf, buflen) <= 0)
{
OK = false;
break;
}
}
}
else
{
bool copydone = false;
while (!interrupted && !copydone)
{ /* for each input line ... */
bool firstload;
bool linedone;
firstload = true;
linedone = false;
while (!linedone)
{ /* for each bufferload in line ... */
int linelen;
char *fgresult;
fgresult = fgets(buf, sizeof(buf), copystream);
if (!fgresult)
{
copydone = true;
break;
}
linelen = strlen(buf);
/* current line is done? */
if (linelen > 0 && buf[linelen - 1] == '\n')
linedone = true;
/* check for EOF marker, but not on a partial line */
if (firstload)
{
if (strcmp(buf, "\\.\n") == 0 ||
strcmp(buf, "\\.\r\n") == 0)
{
copydone = true;
break;
}
firstload = false;
}
if (PQputCopyData(conn, buf, linelen) <= 0)
{
OK = false;
copydone = true;
break;
}
}
}
}
if (interrupted)
{
PQputCopyEnd(conn, "canceled by user");
return PQgetResult(conn);
}
/* Check for read error */
if (ferror(copystream))
OK = false;
/* Terminate data transfer */
if (PQputCopyEnd(conn, OK ? NULL : "aborted because of read failure") <= 0)
OK = false;
/* Check command status and return to normal libpq state */
if (!OK)
return NULL;
return PQgetResult(conn);
}
bool
handleCopyIn(PGconn *conn, FILE *copystream, bool isbinary)
{
bool OK;
const char *prompt;
char buf[COPYBUFSIZ];
PGresult *res;
/*
* Establish longjmp destination for exiting from wait-for-input. (This is
* only effective while sigint_interrupt_enabled is TRUE.)
*/
if (sigsetjmp(sigint_interrupt_jmp, 1) != 0)
{
/* got here with longjmp */
/* Terminate data transfer */
PQputCopyEnd(conn, _("canceled by user"));
OK = false;
goto copyin_cleanup;
}
/* Prompt if interactive input */
if (isatty(fileno(copystream)))
{
if (!pset.quiet)
puts(_("Enter data to be copied followed by a newline.\n"
"End with a backslash and a period on a line by itself."));
prompt = get_prompt(PROMPT_COPY);
}
else
prompt = NULL;
OK = true;
if (isbinary)
{
/* interactive input probably silly, but give one prompt anyway */
if (prompt)
{
fputs(prompt, stdout);
fflush(stdout);
}
for (;;)
{
int buflen;
/* enable longjmp while waiting for input */
sigint_interrupt_enabled = true;
buflen = fread(buf, 1, COPYBUFSIZ, copystream);
sigint_interrupt_enabled = false;
if (buflen <= 0)
break;
if (PQputCopyData(conn, buf, buflen) <= 0)
{
OK = false;
break;
}
}
}
else
{
bool copydone = false;
while (!copydone)
{ /* for each input line ... */
bool firstload;
bool linedone;
if (prompt)
{
fputs(prompt, stdout);
fflush(stdout);
}
firstload = true;
linedone = false;
while (!linedone)
{ /* for each bufferload in line ... */
int linelen;
char *fgresult;
/* enable longjmp while waiting for input */
sigint_interrupt_enabled = true;
fgresult = fgets(buf, sizeof(buf), copystream);
sigint_interrupt_enabled = false;
if (!fgresult)
{
copydone = true;
break;
}
linelen = strlen(buf);
/* current line is done? */
if (linelen > 0 && buf[linelen - 1] == '\n')
linedone = true;
/* check for EOF marker, but not on a partial line */
if (firstload)
{
/*
* This code erroneously assumes '\.' on a line alone
* inside a quoted CSV string terminates the \copy.
* http://www.postgresql.org/message-id/[email protected]
*/
if (strcmp(buf, "\\.\n") == 0 ||
strcmp(buf, "\\.\r\n") == 0)
{
copydone = true;
break;
}
firstload = false;
}
if (PQputCopyData(conn, buf, linelen) <= 0)
{
OK = false;
copydone = true;
break;
}
}
if (copystream == pset.cur_cmd_source)
pset.lineno++;
}
}
/* Check for read error */
if (ferror(copystream))
OK = false;
/* Terminate data transfer */
if (PQputCopyEnd(conn,
OK ? NULL : _("aborted because of read failure")) <= 0)
OK = false;
copyin_cleanup:
/*
* Check command status and return to normal libpq state
*
* We must not ever return with the status still PGRES_COPY_IN. Our
* caller is unable to distinguish that situation from reaching the next
* COPY in a command string that happened to contain two consecutive COPY
* FROM STDIN commands. XXX if something makes PQputCopyEnd() fail
* indefinitely while retaining status PGRES_COPY_IN, we get an infinite
* loop. This is more realistic than handleCopyOut()'s counterpart risk.
*/
while (res = PQgetResult(conn), PQresultStatus(res) == PGRES_COPY_IN)
{
OK = false;
PQclear(res);
PQputCopyEnd(pset.db, _("trying to exit copy mode"));
}
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
psql_error("%s", PQerrorMessage(conn));
OK = false;
}
PQclear(res);
return OK;
}
/* {{{ proto string PDO::pgsqlCopyFromArray(string $table_name , array $rows [, string $delimiter [, string $null_as ] [, string $fields])
Returns true if the copy worked fine or false if error */
static PHP_METHOD(PDO, pgsqlCopyFromArray)
{
pdo_dbh_t *dbh;
pdo_pgsql_db_handle *H;
zval *pg_rows;
char *table_name, *pg_delim = NULL, *pg_null_as = NULL, *pg_fields = NULL;
size_t table_name_len, pg_delim_len = 0, pg_null_as_len = 0, pg_fields_len;
char *query;
PGresult *pgsql_result;
ExecStatusType status;
if (zend_parse_parameters(ZEND_NUM_ARGS(), "s/a|sss",
&table_name, &table_name_len, &pg_rows,
&pg_delim, &pg_delim_len, &pg_null_as, &pg_null_as_len, &pg_fields, &pg_fields_len) == FAILURE) {
return;
}
if (!zend_hash_num_elements(Z_ARRVAL_P(pg_rows))) {
php_error_docref(NULL, E_WARNING, "Cannot copy from an empty array");
RETURN_FALSE;
}
dbh = Z_PDO_DBH_P(getThis());
PDO_CONSTRUCT_CHECK;
PDO_DBH_CLEAR_ERR();
/* using pre-9.0 syntax as PDO_pgsql is 7.4+ compatible */
if (pg_fields) {
spprintf(&query, 0, "COPY %s (%s) FROM STDIN WITH DELIMITER E'%c' NULL AS E'%s'", table_name, pg_fields, (pg_delim_len ? *pg_delim : '\t'), (pg_null_as_len ? pg_null_as : "\\\\N"));
} else {
spprintf(&query, 0, "COPY %s FROM STDIN WITH DELIMITER E'%c' NULL AS E'%s'", table_name, (pg_delim_len ? *pg_delim : '\t'), (pg_null_as_len ? pg_null_as : "\\\\N"));
}
/* Obtain db Handle */
H = (pdo_pgsql_db_handle *)dbh->driver_data;
while ((pgsql_result = PQgetResult(H->server))) {
PQclear(pgsql_result);
}
pgsql_result = PQexec(H->server, query);
efree(query);
query = NULL;
if (pgsql_result) {
status = PQresultStatus(pgsql_result);
} else {
status = (ExecStatusType) PQstatus(H->server);
}
if (status == PGRES_COPY_IN && pgsql_result) {
int command_failed = 0;
int buffer_len = 0;
zval *tmp;
PQclear(pgsql_result);
ZEND_HASH_FOREACH_VAL(Z_ARRVAL_P(pg_rows), tmp) {
int query_len;
convert_to_string_ex(tmp);
if (buffer_len < Z_STRLEN_P(tmp)) {
buffer_len = Z_STRLEN_P(tmp);
query = erealloc(query, buffer_len + 2); /* room for \n\0 */
}
memcpy(query, Z_STRVAL_P(tmp), Z_STRLEN_P(tmp));
query_len = Z_STRLEN_P(tmp);
if (query[query_len - 1] != '\n') {
query[query_len++] = '\n';
}
query[query_len] = '\0';
if (PQputCopyData(H->server, query, query_len) != 1) {
efree(query);
pdo_pgsql_error(dbh, PGRES_FATAL_ERROR, NULL);
PDO_HANDLE_DBH_ERR();
RETURN_FALSE;
}
} ZEND_HASH_FOREACH_END();
SEXP
RS_PostgreSQL_CopyInDataframe(Con_Handle * conHandle, SEXP x, SEXP nrow, SEXP ncol)
{
S_EVALUATOR RS_DBI_connection * con;
int nr, nc, i, j;
const char *cna ="\\N", *tmp=NULL /* -Wall */;
char cdec = '.';
PGconn *my_connection;
int pqretcode;
nr = asInteger(nrow);
nc = asInteger(ncol);
const int buff_threshold = 8000;
con = RS_DBI_getConnection(conHandle);
my_connection = (PGconn *) con->drvConnection;
if(isVectorList(x)) { /* A data frame */
R_StringBuffer rstrbuf = {NULL, 0, 10000};
char *strBuf = Calloc(buff_threshold * 2 + 2, char); /* + 2 for '\t' or '\n' plus '\0'*/
char *strendp = strBuf;
SEXP *levels;
*strendp = '\0';
R_AllocStringBuffer(10000, &rstrbuf);
/* handle factors internally, check integrity */
levels = (SEXP *) R_alloc(nc, sizeof(SEXP));
for(j = 0; j < nc; j++) {
SEXP xj;
xj = VECTOR_ELT(x, j);
if(LENGTH(xj) != nr)
error(("corrupt data frame -- length of column %d does not not match nrows"), j+1);
if(inherits(xj, "factor")) {
levels[j] = getAttrib(xj, R_LevelsSymbol);
} else levels[j] = R_NilValue;
}
for(i = 0; i < nr; i++) {
for(j = 0; j < nc; j++) {
SEXP xj;
xj = VECTOR_ELT(x, j);
if(j > 0){
*strendp++ = '\t';/*need no size count check here*/
}
if(isna(xj, i)) tmp = cna;
else {
if(!isNull(levels[j])) {
/* We cannot assume factors have integer levels */
if(TYPEOF(xj) == INTSXP){
tmp = EncodeElementS(levels[j], INTEGER(xj)[i] - 1,
&rstrbuf, cdec);
}else if(TYPEOF(xj) == REALSXP){
tmp = EncodeElementS(levels[j], REAL(xj)[i] - 1,
&rstrbuf, cdec);
}else
error("column %s claims to be a factor but does not have numeric codes", j+1);
} else {
tmp = EncodeElementS(xj, i,
&rstrbuf, cdec);
}
}
{
size_t n;
size_t len = strendp - strBuf;
n = strlen(tmp);
if (len + n < buff_threshold){
memcpy(strendp, tmp, n);/* we already know the length */
strendp += n;
}else if(n < buff_threshold){ /*copy and flush*/
memcpy(strendp, tmp, n);/* we already know the length */
pqretcode = PQputCopyData(my_connection, strBuf, len + n);
chkpqcopydataerr(my_connection, pqretcode);
strendp = strBuf;
}else{ /*flush and copy current*/
if(len > 0){
pqretcode = PQputCopyData(my_connection, strBuf, len);
chkpqcopydataerr(my_connection, pqretcode);
strendp = strBuf;
}
pqretcode = PQputCopyData(my_connection, tmp, n);
chkpqcopydataerr(my_connection, pqretcode);
}
}
}
*strendp = '\n'; strendp +=1; *strendp='\0';
}
pqretcode = PQputCopyData(my_connection, strBuf, strendp - strBuf);
chkpqcopydataerr(my_connection, pqretcode);
Free(strBuf);
R_FreeStringBuffer(&rstrbuf);
}
PQputCopyEnd(my_connection, NULL);
return R_NilValue;
}
bool
handleCopyIn(PGconn *conn, FILE *copystream, bool isbinary)
{
bool OK;
const char* prompt;
char buf[COPYBUFSIZ];
PGresult* res;
/*
* Establish longjmp destination for exiting from wait-for-input. (This is
* only effective while sigint_interrupt_enabled is TRUE.)
*/
if (sigsetjmp(sigint_interrupt_jmp, 1) != 0) {
/* got here with longjmp */
/* Terminate data transfer */
PQputCopyEnd(conn, _("canceled by user"));
/* Check command status and return to normal libpq state */
res = PQgetResult(conn);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
psql_error("%s", PQerrorMessage(conn));
PQclear(res);
return false;
}
/* Prompt if interactive input */
if (isatty(fileno(copystream))) {
if (!pset.quiet)
puts(_("Enter data to be copied followed by a newline.\n"
"End with a backslash and a period on a line by itself."));
prompt = get_prompt(PROMPT_COPY);
} else {
prompt = NULL;
}
OK = true;
if (isbinary) {
/* interactive input probably silly, but give one prompt anyway */
if (prompt) {
fputs(prompt, stdout);
fflush(stdout);
}
for (;;) {
int buflen;
/* enable longjmp while waiting for input */
sigint_interrupt_enabled = true;
buflen = fread(buf, 1, COPYBUFSIZ, copystream);
sigint_interrupt_enabled = false;
if (buflen <= 0)
break;
if (PQputCopyData(conn, buf, buflen) <= 0) {
OK = false;
break;
}
}
} else {
bool copydone = false;
while (!copydone) {
/* for each input line ... */
bool firstload;
bool linedone;
if (prompt) {
fputs(prompt, stdout);
fflush(stdout);
}
firstload = true;
linedone = false;
while (!linedone) {
/* for each bufferload in line ... */
int linelen;
char* fgresult;
/* enable longjmp while waiting for input */
sigint_interrupt_enabled = true;
fgresult = fgets(buf, sizeof(buf), copystream);
sigint_interrupt_enabled = false;
if (!fgresult) {
copydone = true;
break;
}
linelen = strlen(buf);
/* current line is done? */
if (linelen > 0 && buf[linelen - 1] == '\n')
linedone = true;
/* check for EOF marker, but not on a partial line */
if (firstload) {
if (strcmp(buf, "\\.\n") == 0 ||
strcmp(buf, "\\.\r\n") == 0) {
copydone = true;
break;
}
firstload = false;
}
if (PQputCopyData(conn, buf, linelen) <= 0) {
OK = false;
copydone = true;
break;
}
}
if (copystream == pset.cur_cmd_source)
pset.lineno++;
}
}
/* Check for read error */
if (ferror(copystream))
OK = false;
/* Terminate data transfer */
if (PQputCopyEnd(conn, OK ? NULL : _("aborted because of read failure")) <= 0)
OK = false;
/* Check command status and return to normal libpq state */
res = PQgetResult(conn);
if (PQresultStatus(res) != PGRES_COMMAND_OK) {
psql_error("%s", PQerrorMessage(conn));
OK = false;
}
PQclear(res);
return OK;
}
bool
handleCopyIn(PGconn *conn, FILE *copystream, bool isbinary, PGresult **res)
{
bool OK;
const char *prompt;
char buf[COPYBUFSIZ];
/*
* Establish longjmp destination for exiting from wait-for-input. (This is
* only effective while sigint_interrupt_enabled is TRUE.)
*/
if (sigsetjmp(sigint_interrupt_jmp, 1) != 0)
{
/* got here with longjmp */
/* Terminate data transfer */
PQputCopyEnd(conn,
(PQprotocolVersion(conn) < 3) ? NULL :
_("canceled by user"));
OK = false;
goto copyin_cleanup;
}
/* Prompt if interactive input */
if (isatty(fileno(copystream)))
{
if (!pset.quiet)
puts(_("Enter data to be copied followed by a newline.\n"
"End with a backslash and a period on a line by itself."));
prompt = get_prompt(PROMPT_COPY);
}
else
prompt = NULL;
OK = true;
if (isbinary)
{
/* interactive input probably silly, but give one prompt anyway */
if (prompt)
{
fputs(prompt, stdout);
fflush(stdout);
}
for (;;)
{
int buflen;
/* enable longjmp while waiting for input */
sigint_interrupt_enabled = true;
buflen = fread(buf, 1, COPYBUFSIZ, copystream);
sigint_interrupt_enabled = false;
if (buflen <= 0)
break;
if (PQputCopyData(conn, buf, buflen) <= 0)
{
OK = false;
break;
}
}
}
else
{
bool copydone = false;
while (!copydone)
{ /* for each input line ... */
bool firstload;
bool linedone;
if (prompt)
{
fputs(prompt, stdout);
fflush(stdout);
}
firstload = true;
linedone = false;
while (!linedone)
{ /* for each bufferload in line ... */
int linelen;
char *fgresult;
/* enable longjmp while waiting for input */
sigint_interrupt_enabled = true;
fgresult = fgets(buf, sizeof(buf), copystream);
sigint_interrupt_enabled = false;
if (!fgresult)
{
copydone = true;
break;
}
linelen = strlen(buf);
/* current line is done? */
if (linelen > 0 && buf[linelen - 1] == '\n')
linedone = true;
/* check for EOF marker, but not on a partial line */
if (firstload)
{
/*
* This code erroneously assumes '\.' on a line alone
* inside a quoted CSV string terminates the \copy.
* http://www.postgresql.org/message-id/E1TdNVQ-0001ju-GO@w
* rigleys.postgresql.org
*/
if (strcmp(buf, "\\.\n") == 0 ||
strcmp(buf, "\\.\r\n") == 0)
{
copydone = true;
break;
}
firstload = false;
}
if (PQputCopyData(conn, buf, linelen) <= 0)
{
OK = false;
copydone = true;
break;
}
}
if (copystream == pset.cur_cmd_source)
pset.lineno++;
}
}
/* Check for read error */
if (ferror(copystream))
OK = false;
/*
* Terminate data transfer. We can't send an error message if we're using
* protocol version 2.
*/
if (PQputCopyEnd(conn,
(OK || PQprotocolVersion(conn) < 3) ? NULL :
_("aborted because of read failure")) <= 0)
OK = false;
copyin_cleanup:
/*
* Check command status and return to normal libpq state.
*
* We do not want to return with the status still PGRES_COPY_IN: our
* caller would be unable to distinguish that situation from reaching the
* next COPY in a command string that happened to contain two consecutive
* COPY FROM STDIN commands. We keep trying PQputCopyEnd() in the hope
* it'll work eventually. (What's actually likely to happen is that in
* attempting to flush the data, libpq will eventually realize that the
* connection is lost. But that's fine; it will get us out of COPY_IN
* state, which is what we need.)
*/
while (*res = PQgetResult(conn), PQresultStatus(*res) == PGRES_COPY_IN)
{
OK = false;
PQclear(*res);
/* We can't send an error message if we're using protocol version 2 */
PQputCopyEnd(conn,
(PQprotocolVersion(conn) < 3) ? NULL :
_("trying to exit copy mode"));
}
if (PQresultStatus(*res) != PGRES_COMMAND_OK)
{
psql_error("%s", PQerrorMessage(conn));
OK = false;
}
return OK;
}
/*
* Receive a log stream starting at the specified position.
*
* If sysidentifier is specified, validate that both the system
* identifier and the timeline matches the specified ones
* (by sending an extra IDENTIFY_SYSTEM command)
*
* All received segments will be written to the directory
* specified by basedir.
*
* The stream_stop callback will be called every time data
* is received, and whenever a segment is completed. If it returns
* true, the streaming will stop and the function
* return. As long as it returns false, streaming will continue
* indefinitely.
*
* standby_message_timeout controls how often we send a message
* back to the master letting it know our progress, in seconds.
* This message will only contain the write location, and never
* flush or replay.
*
* Note: The log position *must* be at a log segment start!
*/
bool
ReceiveXlogStream(PGconn *conn, XLogRecPtr startpos, uint32 timeline,
char *sysidentifier, char *basedir,
stream_stop_callback stream_stop,
int standby_message_timeout, bool rename_partial)
{
char query[128];
char current_walfile_name[MAXPGPATH];
PGresult *res;
char *copybuf = NULL;
int64 last_status = -1;
XLogRecPtr blockpos = InvalidXLogRecPtr;
if (sysidentifier != NULL)
{
/* Validate system identifier and timeline hasn't changed */
res = PQexec(conn, "IDENTIFY_SYSTEM");
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
fprintf(stderr,
_("%s: could not send replication command \"%s\": %s"),
progname, "IDENTIFY_SYSTEM", PQerrorMessage(conn));
PQclear(res);
return false;
}
if (PQnfields(res) != 3 || PQntuples(res) != 1)
{
fprintf(stderr,
_("%s: could not identify system: got %d rows and %d fields, expected %d rows and %d fields\n"),
progname, PQntuples(res), PQnfields(res), 1, 3);
PQclear(res);
return false;
}
if (strcmp(sysidentifier, PQgetvalue(res, 0, 0)) != 0)
{
fprintf(stderr,
_("%s: system identifier does not match between base backup and streaming connection\n"),
progname);
PQclear(res);
return false;
}
if (timeline != atoi(PQgetvalue(res, 0, 1)))
{
fprintf(stderr,
_("%s: timeline does not match between base backup and streaming connection\n"),
progname);
PQclear(res);
return false;
}
PQclear(res);
}
/* Initiate the replication stream at specified location */
snprintf(query, sizeof(query), "START_REPLICATION %X/%X", startpos.xlogid, startpos.xrecoff);
res = PQexec(conn, query);
if (PQresultStatus(res) != PGRES_COPY_BOTH)
{
fprintf(stderr, _("%s: could not send replication command \"%s\": %s"),
progname, "START_REPLICATION", PQresultErrorMessage(res));
PQclear(res);
return false;
}
PQclear(res);
/*
* Receive the actual xlog data
*/
while (1)
{
int r;
int xlogoff;
int bytes_left;
int bytes_written;
int64 now;
if (copybuf != NULL)
{
PQfreemem(copybuf);
copybuf = NULL;
}
/*
* Check if we should continue streaming, or abort at this point.
*/
if (stream_stop && stream_stop(blockpos, timeline, false))
{
if (walfile != -1 && !close_walfile(basedir, current_walfile_name,
rename_partial))
/* Potential error message is written by close_walfile */
goto error;
return true;
}
/*
* Potentially send a status message to the master
*/
now = localGetCurrentTimestamp();
if (standby_message_timeout > 0 &&
localTimestampDifferenceExceeds(last_status, now,
standby_message_timeout))
{
/* Time to send feedback! */
char replybuf[sizeof(StandbyReplyMessage) + 1];
StandbyReplyMessage *replymsg;
replymsg = (StandbyReplyMessage *) (replybuf + 1);
replymsg->write = blockpos;
replymsg->flush = InvalidXLogRecPtr;
replymsg->apply = InvalidXLogRecPtr;
replymsg->sendTime = now;
replybuf[0] = 'r';
if (PQputCopyData(conn, replybuf, sizeof(replybuf)) <= 0 ||
PQflush(conn))
{
fprintf(stderr, _("%s: could not send feedback packet: %s"),
progname, PQerrorMessage(conn));
goto error;
}
last_status = now;
}
r = PQgetCopyData(conn, ©buf, 1);
if (r == 0)
{
/*
* In async mode, and no data available. We block on reading but
* not more than the specified timeout, so that we can send a
* response back to the client.
*/
fd_set input_mask;
struct timeval timeout;
struct timeval *timeoutptr;
FD_ZERO(&input_mask);
FD_SET(PQsocket(conn), &input_mask);
if (standby_message_timeout)
{
TimestampTz targettime;
long secs;
int usecs;
targettime = TimestampTzPlusMilliseconds(last_status,
standby_message_timeout - 1);
localTimestampDifference(now,
targettime,
&secs,
&usecs);
if (secs <= 0)
timeout.tv_sec = 1; /* Always sleep at least 1 sec */
else
timeout.tv_sec = secs;
timeout.tv_usec = usecs;
timeoutptr = &timeout;
}
else
timeoutptr = NULL;
r = select(PQsocket(conn) + 1, &input_mask, NULL, NULL, timeoutptr);
if (r == 0 || (r < 0 && errno == EINTR))
{
/*
* Got a timeout or signal. Continue the loop and either
* deliver a status packet to the server or just go back into
* blocking.
*/
continue;
}
else if (r < 0)
{
fprintf(stderr, _("%s: select() failed: %s\n"),
progname, strerror(errno));
goto error;
}
/* Else there is actually data on the socket */
if (PQconsumeInput(conn) == 0)
{
fprintf(stderr,
_("%s: could not receive data from WAL stream: %s"),
progname, PQerrorMessage(conn));
goto error;
}
continue;
}
if (r == -1)
/* End of copy stream */
break;
if (r == -2)
{
fprintf(stderr, _("%s: could not read COPY data: %s"),
progname, PQerrorMessage(conn));
goto error;
}
if (copybuf[0] == 'k')
{
/*
* keepalive message, sent in 9.2 and newer. We just ignore this
* message completely, but need to skip past it in the stream.
*/
if (r != STREAMING_KEEPALIVE_SIZE)
{
fprintf(stderr,
_("%s: keepalive message has incorrect size %d\n"),
progname, r);
goto error;
}
continue;
}
else if (copybuf[0] != 'w')
{
fprintf(stderr, _("%s: unrecognized streaming header: \"%c\"\n"),
progname, copybuf[0]);
goto error;
}
if (r < STREAMING_HEADER_SIZE + 1)
{
fprintf(stderr, _("%s: streaming header too small: %d\n"),
progname, r);
goto error;
}
/* Extract WAL location for this block */
memcpy(&blockpos, copybuf + 1, 8);
xlogoff = blockpos.xrecoff % XLOG_SEG_SIZE;
/*
* Verify that the initial location in the stream matches where we
* think we are.
*/
if (walfile == -1)
{
/* No file open yet */
if (xlogoff != 0)
{
fprintf(stderr,
_("%s: received transaction log record for offset %u with no file open\n"),
progname, xlogoff);
goto error;
}
}
else
{
/* More data in existing segment */
/* XXX: store seek value don't reseek all the time */
if (lseek(walfile, 0, SEEK_CUR) != xlogoff)
{
fprintf(stderr,
_("%s: got WAL data offset %08x, expected %08x\n"),
progname, xlogoff, (int) lseek(walfile, 0, SEEK_CUR));
goto error;
}
}
bytes_left = r - STREAMING_HEADER_SIZE;
bytes_written = 0;
while (bytes_left)
{
int bytes_to_write;
/*
* If crossing a WAL boundary, only write up until we reach
* XLOG_SEG_SIZE.
*/
if (xlogoff + bytes_left > XLOG_SEG_SIZE)
bytes_to_write = XLOG_SEG_SIZE - xlogoff;
else
bytes_to_write = bytes_left;
if (walfile == -1)
{
walfile = open_walfile(blockpos, timeline,
basedir, current_walfile_name);
if (walfile == -1)
/* Error logged by open_walfile */
goto error;
}
if (write(walfile,
copybuf + STREAMING_HEADER_SIZE + bytes_written,
bytes_to_write) != bytes_to_write)
{
fprintf(stderr,
_("%s: could not write %u bytes to WAL file \"%s\": %s\n"),
progname, bytes_to_write, current_walfile_name,
strerror(errno));
goto error;
}
/* Write was successful, advance our position */
bytes_written += bytes_to_write;
bytes_left -= bytes_to_write;
XLByteAdvance(blockpos, bytes_to_write);
xlogoff += bytes_to_write;
/* Did we reach the end of a WAL segment? */
if (blockpos.xrecoff % XLOG_SEG_SIZE == 0)
{
if (!close_walfile(basedir, current_walfile_name, false))
/* Error message written in close_walfile() */
goto error;
xlogoff = 0;
if (stream_stop != NULL)
{
/*
* Callback when the segment finished, and return if it
* told us to.
*/
if (stream_stop(blockpos, timeline, true))
return true;
}
}
}
/* No more data left to write, start receiving next copy packet */
}
/*
* The only way to get out of the loop is if the server shut down the
* replication stream. If it's a controlled shutdown, the server will send
* a shutdown message, and we'll return the latest xlog location that has
* been streamed.
*/
res = PQgetResult(conn);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr,
_("%s: unexpected termination of replication stream: %s"),
progname, PQresultErrorMessage(res));
goto error;
}
PQclear(res);
/* Complain if we've not reached stop point yet */
if (stream_stop != NULL && !stream_stop(blockpos, timeline, false))
{
fprintf(stderr, _("%s: replication stream was terminated before stop point\n"),
progname);
goto error;
}
if (copybuf != NULL)
PQfreemem(copybuf);
if (walfile != -1 && close(walfile) != 0)
fprintf(stderr, _("%s: could not close file \"%s\": %s\n"),
progname, current_walfile_name, strerror(errno));
walfile = -1;
return true;
error:
if (copybuf != NULL)
PQfreemem(copybuf);
if (walfile != -1 && close(walfile) != 0)
fprintf(stderr, _("%s: could not close file \"%s\": %s\n"),
progname, current_walfile_name, strerror(errno));
walfile = -1;
return false;
}
