static void add_projection_desc_httpheader(CHURL_HEADERS headers, ProjectionInfo *projInfo, List *qualsAttributes) {
int i;
char long_number[sizeof(int32) * 8];
int *varNumbers = projInfo->pi_varNumbers;
StringInfoData formatter;
initStringInfo(&formatter);
/* Convert the number of projection columns to a string */
pg_ltoa(list_length(projInfo->pi_targetlist) + list_length(qualsAttributes), long_number);
churl_headers_append(headers, "X-GP-ATTRS-PROJ", long_number);
for(i = 0; i < list_length(projInfo->pi_targetlist); i++) {
int number = varNumbers[i] - 1;
pg_ltoa(number, long_number);
resetStringInfo(&formatter);
appendStringInfo(&formatter, "X-GP-ATTRS-PROJ-IDX");
churl_headers_append(headers, formatter.data,long_number);
}
ListCell *attribute = NULL;
foreach(attribute, qualsAttributes)
{
AttrNumber attrNumber = lfirst_int(attribute);
pg_ltoa(attrNumber, long_number);
resetStringInfo(&formatter);
appendStringInfo(&formatter, "X-GP-ATTRS-PROJ-IDX");
churl_headers_append(headers, formatter.data,long_number);
}
CHURL_HANDLE churl_init_upload(const char* url, CHURL_HEADERS headers)
{
churl_context* context = churl_new_context();
create_curl_handle(context);
context->upload = true;
clear_error_buffer(context);
set_curl_option(context, CURLOPT_URL, url);
set_curl_option(context, CURLOPT_VERBOSE, (const void*)FALSE);
set_curl_option(context, CURLOPT_ERRORBUFFER, context->curl_error_buffer);
set_curl_option(context, CURLOPT_IPRESOLVE, (const void*)CURL_IPRESOLVE_V4);
set_curl_option(context, CURLOPT_POST, (const void*)TRUE);
set_curl_option(context, CURLOPT_READFUNCTION, read_callback);
set_curl_option(context, CURLOPT_READDATA, context);
set_curl_option(context, CURLOPT_WRITEFUNCTION, write_callback);
set_curl_option(context, CURLOPT_WRITEDATA, context);
set_curl_option(context, CURLOPT_HEADERFUNCTION, header_callback);
set_curl_option(context, CURLOPT_HEADERDATA, context);
churl_headers_append(headers, "Content-Type", "application/octet-stream");
churl_headers_append(headers, "Transfer-Encoding", "chunked");
churl_headers_append(headers, "Expect", "100-continue");
churl_headers_set(context, headers);
print_http_headers(headers);
setup_multi_handle(context);
return (CHURL_HANDLE)context;
}
CHURL_HANDLE churl_init_upload(const char* url, CHURL_HEADERS headers)
{
churl_context* context = churl_init(url, headers);
context->upload = true;
set_curl_option(context, CURLOPT_POST, (const void*) TRUE);
set_curl_option(context, CURLOPT_READFUNCTION, read_callback);
set_curl_option(context, CURLOPT_READDATA, context);
churl_headers_append(headers, "Content-Type", "application/octet-stream");
churl_headers_append(headers, "Transfer-Encoding", "chunked");
churl_headers_append(headers, "Expect", "100-continue");
return (CHURL_HANDLE)context;
}
/*
* get list of data nodes' rest servers,
* and choose one (based on modulo segment id).
* if pxf_isilon is true, there are no PXF instances on the datanodes.
* TODO: add locality
*/
PxfServer* get_pxf_server(GPHDUri* gphd_uri, const Relation rel)
{
ClientContext client_context; /* holds the communication info */
PxfInputData inputData = {0};
List *rest_servers = NIL;
ListCell *rest_server_c = NULL;
PxfServer *found_server = NULL;
PxfServer *ret_server = (PxfServer*)palloc0(sizeof(PxfServer));
char *server_ip = NULL;
int server_index = 0;
Assert(gphd_uri);
/* init context */
init_client_context(&client_context);
client_context.http_headers = churl_headers_init();
/* set HTTP header that guarantees response in JSON format */
churl_headers_append(client_context.http_headers, REST_HEADER_JSON_RESPONSE, NULL);
if (!client_context.http_headers)
{
return NULL;
}
/*
* Enrich the curl HTTP header
*/
inputData.headers = client_context.http_headers;
inputData.gphduri = gphd_uri;
inputData.rel = rel;
inputData.filterstr = NULL; /* We do not supply filter data to the HTTP header */
add_delegation_token(&inputData);
build_http_header(&inputData);
int port = atoi(gphd_uri->port);
if (!pxf_isilon)
{
/* send request */
rest_servers = get_datanode_rest_servers(gphd_uri, &client_context);
foreach(rest_server_c, rest_servers)
{
PxfServer *rest_server = (PxfServer*)lfirst(rest_server_c);
/* In case there are several rest servers on the same host, we assume
* there are multiple DN residing together.
* The port is incremented by one, to match singlecluster convention */
if (pxf_service_singlecluster)
{
if (server_ip == NULL)
{
server_ip = rest_server->host;
}
else if (are_ips_equal(server_ip, rest_server->host))
{
port++;
}
}
rest_server->port = port;
}
void churl_headers_override(CHURL_HEADERS headers,
const char* key,
const char* value)
{
churl_settings* settings = (churl_settings*)headers;
struct curl_slist* header_cell = settings->headers;
char* key_option = NULL;
char* header_data = NULL;
/* key must not be empty */
Assert(key != NULL);
/* key to compare with in the headers */
key_option = build_header_str("%s:%s", key, value ? "" : NULL);
/* find key in headers list */
while (header_cell != NULL)
{
header_data = header_cell->data;
if (strncmp(key_option, header_data, strlen(key_option)) == 0)
{
elog(DEBUG2, "churl_headers_override: Found existing header %s with key %s (for new value %s)",
header_data, key_option, value);
break;
}
header_cell = header_cell->next;
}
if (header_cell != NULL) /* found key */
{
char* new_data = build_header_str("%s: %s", key, value);
char* old_data = header_cell->data;
header_cell->data = strdup(new_data);
elog(DEBUG4, "churl_headers_override: new data: %s, old data: %s", new_data, old_data);
free(old_data);
pfree(new_data);
}
else
{
churl_headers_append(headers, key, value);
}
pfree(key_option);
}
void append_churl_header_if_exists(gphadoop_context* context, const char* key, const char* value)
{
if (value)
churl_headers_append(context->churl_headers, key, value);
}
/*
* Add key/value pairs to connection header.
* These values are the context of the query and used
* by the remote component.
*/
void build_http_header(PxfInputData *input)
{
extvar_t ev;
CHURL_HEADERS headers = input->headers;
GPHDUri *gphduri = input->gphduri;
Relation rel = input->rel;
char *filterstr = input->filterstr;
ProjectionInfo *proj_info = input->proj_info;
if (rel != NULL)
{
/* format */
ExtTableEntry *exttbl = GetExtTableEntry(rel->rd_id);
/* pxf treats CSV as TEXT */
char* format = get_format_name(exttbl->fmtcode);
churl_headers_append(headers, "X-GP-FORMAT", format);
/* Record fields - name and type of each field */
add_tuple_desc_httpheader(headers, rel);
}
if (proj_info != NULL && proj_info->pi_isVarList)
{
List* qualsAttributes = extractPxfAttributes(input->quals);
/* projection information is incomplete if columns from WHERE clause wasn't extracted */
if (qualsAttributes != NIL || list_length(input->quals) == 0)
{
add_projection_desc_httpheader(headers, proj_info, qualsAttributes);
}
else
elog(DEBUG2, "Query will not be optimized to use projection information");
}
/* GP cluster configuration */
external_set_env_vars(&ev, gphduri->uri, false, NULL, NULL, false, 0);
churl_headers_append(headers, "X-GP-SEGMENT-ID", ev.GP_SEGMENT_ID);
churl_headers_append(headers, "X-GP-SEGMENT-COUNT", ev.GP_SEGMENT_COUNT);
churl_headers_append(headers, "X-GP-XID", ev.GP_XID);
/* Report alignment size to remote component
* GPDBWritable uses alignment that has to be the same as
* in the C code.
* Since the C code can be compiled for both 32 and 64 bits,
* the alignment can be either 4 or 8.
*/
add_alignment_size_httpheader(headers);
/* headers for uri data */
churl_headers_append(headers, "X-GP-URL-HOST", gphduri->host);
churl_headers_append(headers, "X-GP-URL-PORT", gphduri->port);
churl_headers_append(headers, "X-GP-DATA-DIR", gphduri->data);
/* location options */
add_location_options_httpheader(headers, gphduri);
/* full uri */
churl_headers_append(headers, "X-GP-URI", gphduri->uri);
/* filters */
if (filterstr)
{
churl_headers_append(headers, "X-GP-HAS-FILTER", "1");
churl_headers_append(headers, "X-GP-FILTER", filterstr);
}
else
churl_headers_append(headers, "X-GP-HAS-FILTER", "0");
add_delegation_token_headers(headers, input);
add_remote_credentials(headers);
}
/*
* Report tuple description to remote component
* Currently, number of attributes, attributes names, types and types modifiers
* Each attribute has a pair of key/value
* where X is the number of the attribute
* X-GP-ATTR-NAMEX - attribute X's name
* X-GP-ATTR-TYPECODEX - attribute X's type OID (e.g, 16)
* X-GP-ATTR-TYPENAMEX - attribute X's type name (e.g, "boolean")
* optional - X-GP-ATTR-TYPEMODX-COUNT - total number of modifier for attribute X
* optional - X-GP-ATTR-TYPEMODX-Y - attribute X's modifiers Y (types which have precision info, like numeric(p,s))
*/
static void add_tuple_desc_httpheader(CHURL_HEADERS headers, Relation rel)
{
char long_number[INT32_CHAR_SIZE];
StringInfoData formatter;
TupleDesc tuple;
initStringInfo(&formatter);
/* Get tuple description itself */
tuple = RelationGetDescr(rel);
/* Convert the number of attributes to a string */
pg_ltoa(tuple->natts, long_number);
churl_headers_append(headers, "X-GP-ATTRS", long_number);
/* Iterate attributes */
for (int i = 0; i < tuple->natts; ++i)
{
/* Add a key/value pair for attribute name */
resetStringInfo(&formatter);
appendStringInfo(&formatter, "X-GP-ATTR-NAME%u", i);
churl_headers_append(headers, formatter.data, tuple->attrs[i]->attname.data);
/* Add a key/value pair for attribute type */
resetStringInfo(&formatter);
appendStringInfo(&formatter, "X-GP-ATTR-TYPECODE%u", i);
pg_ltoa(tuple->attrs[i]->atttypid, long_number);
churl_headers_append(headers, formatter.data, long_number);
/* Add a key/value pair for attribute type name */
resetStringInfo(&formatter);
appendStringInfo(&formatter, "X-GP-ATTR-TYPENAME%u", i);
churl_headers_append(headers, formatter.data, TypeOidGetTypename(tuple->attrs[i]->atttypid));
/* Add attribute type modifiers if any*/
if (tuple->attrs[i]->atttypmod > -1)
{
switch (tuple->attrs[i]->atttypid)
{
case NUMERICOID:
{
resetStringInfo(&formatter);
appendStringInfo(&formatter, "X-GP-ATTR-TYPEMOD%u-COUNT", i);
pg_ltoa(2, long_number);
churl_headers_append(headers, formatter.data, long_number);
/* precision */
resetStringInfo(&formatter);
appendStringInfo(&formatter, "X-GP-ATTR-TYPEMOD%u-%u", i, 0);
pg_ltoa((tuple->attrs[i]->atttypmod >> 16) & 0xffff, long_number);
churl_headers_append(headers, formatter.data, long_number);
/* scale */
resetStringInfo(&formatter);
appendStringInfo(&formatter, "X-GP-ATTR-TYPEMOD%u-%u", i, 1);
pg_ltoa((tuple->attrs[i]->atttypmod - VARHDRSZ) & 0xffff, long_number);
churl_headers_append(headers, formatter.data, long_number);
break;
}
case CHAROID:
case BPCHAROID:
case VARCHAROID:
{
resetStringInfo(&formatter);
appendStringInfo(&formatter, "X-GP-ATTR-TYPEMOD%u-COUNT", i);
pg_ltoa(1, long_number);
churl_headers_append(headers, formatter.data, long_number);
resetStringInfo(&formatter);
appendStringInfo(&formatter, "X-GP-ATTR-TYPEMOD%u-%u", i, 0);
pg_ltoa((tuple->attrs[i]->atttypmod - VARHDRSZ), long_number);
churl_headers_append(headers, formatter.data, long_number);
break;
}
case VARBITOID:
case BITOID:
case TIMESTAMPOID:
case TIMESTAMPTZOID:
case TIMEOID:
case TIMETZOID:
{
resetStringInfo(&formatter);
appendStringInfo(&formatter, "X-GP-ATTR-TYPEMOD%u-COUNT", i);
pg_ltoa(1, long_number);
churl_headers_append(headers, formatter.data, long_number);
resetStringInfo(&formatter);
appendStringInfo(&formatter, "X-GP-ATTR-TYPEMOD%u-%u", i, 0);
pg_ltoa((tuple->attrs[i]->atttypmod), long_number);
churl_headers_append(headers, formatter.data, long_number);
break;
}
case INTERVALOID:
{
resetStringInfo(&formatter);
appendStringInfo(&formatter, "X-GP-ATTR-TYPEMOD%u-COUNT", i);
pg_ltoa(1, long_number);
churl_headers_append(headers, formatter.data, long_number);
resetStringInfo(&formatter);
appendStringInfo(&formatter, "X-GP-ATTR-TYPEMOD%u-%u", i, 0);
pg_ltoa(INTERVAL_PRECISION(tuple->attrs[i]->atttypmod), long_number);
churl_headers_append(headers, formatter.data, long_number);
break;
}
default:
elog(DEBUG5, "add_tuple_desc_httpheader: unsupported type %d ", tuple->attrs[i]->atttypid);
break;
}
}
}
pfree(formatter.data);
}
/* Report alignment size to remote component
* GPDBWritable uses alignment that has to be the same as
* in the C code.
* Since the C code can be compiled for both 32 and 64 bits,
* the alignment can be either 4 or 8.
*/
static void add_alignment_size_httpheader(CHURL_HEADERS headers)
{
char tmp[sizeof(char*)];
pg_ltoa(sizeof(char*), tmp);
churl_headers_append(headers, "X-GP-ALIGNMENT", tmp);
}
