static void on_connection_ready(struct st_h2o_http1client_t *client)
{
h2o_iovec_t proxy_protocol = h2o_iovec_init(NULL, 0);
int chunked = 0;
h2o_iovec_t connection_header = h2o_iovec_init(NULL, 0);
h2o_httpclient_properties_t props = {
&proxy_protocol, &chunked, &connection_header,
};
h2o_iovec_t method;
h2o_url_t url;
h2o_header_t *headers;
size_t num_headers;
h2o_iovec_t body;
client->super._cb.on_head = client->super._cb.on_connect(&client->super, NULL, &method, &url, (const h2o_header_t **)&headers,
&num_headers, &body, &client->proceed_req, &props, client->_origin);
if (client->super._cb.on_head == NULL) {
close_client(client);
return;
}
h2o_iovec_t reqbufs[3];
size_t reqbufcnt = 0;
if (props.proxy_protocol->base != NULL)
reqbufs[reqbufcnt++] = *props.proxy_protocol;
reqbufs[reqbufcnt++] = build_request(client, method, url, *props.connection_header, headers, num_headers);
client->_is_chunked = *props.chunked;
client->_method_is_head = h2o_memis(method.base, method.len, H2O_STRLIT("HEAD"));
if (client->proceed_req != NULL) {
if (body.base != NULL) {
h2o_buffer_init(&client->_body_buf, &h2o_socket_buffer_prototype);
if (h2o_buffer_append(&client->_body_buf, body.base, body.len) == 0) {
on_send_request(client->sock, "Internal error");
return;
}
}
h2o_socket_write(client->sock, reqbufs, reqbufcnt, on_req_body_done);
} else {
if (client->_is_chunked) {
assert(body.base != NULL);
reqbufcnt += encode_chunk(client, reqbufs + reqbufcnt, body);
} else if (body.base != NULL) {
reqbufs[reqbufcnt++] = body;
}
h2o_socket_write(client->sock, reqbufs, reqbufcnt, on_send_request);
}
/* TODO no need to set the timeout if all data has been written into TCP sendbuf */
client->super._timeout.cb = on_send_timeout;
h2o_timer_link(client->super.ctx->loop, client->super.ctx->io_timeout, &client->super._timeout);
client->super.timings.request_begin_at = h2o_gettimeofday(client->super.ctx->loop);
}
void ChunkEncoder::encode_all_chunks(const std::shared_ptr<Table>& table,
const SegmentEncodingSpec& segment_encoding_spec) {
const auto column_types = table->column_data_types();
for (ChunkID chunk_id{0}; chunk_id < table->chunk_count(); ++chunk_id) {
auto chunk = table->get_chunk(chunk_id);
encode_chunk(chunk, column_types, segment_encoding_spec);
}
}
void ChunkEncoder::encode_all_chunks(const std::shared_ptr<Table>& table,
const ChunkEncodingSpec& chunk_encoding_spec) {
Assert(chunk_encoding_spec.size() == table->column_count(),
"Number of encoding specs must match table’s column count.");
const auto column_types = table->column_data_types();
for (ChunkID chunk_id{0}; chunk_id < table->chunk_count(); ++chunk_id) {
auto chunk = table->get_chunk(chunk_id);
encode_chunk(chunk, column_types, chunk_encoding_spec);
}
}
void ChunkEncoder::encode_chunks(const std::shared_ptr<Table>& table, const std::vector<ChunkID>& chunk_ids,
const SegmentEncodingSpec& segment_encoding_spec) {
const auto column_data_types = table->column_data_types();
for (auto chunk_id : chunk_ids) {
Assert(chunk_id < table->chunk_count(), "Chunk with given ID does not exist.");
auto chunk = table->get_chunk(chunk_id);
encode_chunk(chunk, column_data_types, segment_encoding_spec);
}
}
void ChunkEncoder::encode_all_chunks(const std::shared_ptr<Table>& table,
const std::vector<ChunkEncodingSpec>& chunk_encoding_specs) {
const auto column_types = table->column_data_types();
const auto chunk_count = static_cast<size_t>(table->chunk_count());
Assert(chunk_encoding_specs.size() == chunk_count, "Number of encoding specs must match table’s chunk count.");
for (ChunkID chunk_id{0}; chunk_id < table->chunk_count(); ++chunk_id) {
auto chunk = table->get_chunk(chunk_id);
const auto chunk_encoding_spec = chunk_encoding_specs[chunk_id];
encode_chunk(chunk, column_types, chunk_encoding_spec);
}
}
static void
encode_bean (GString *g, gpointer bean)
{
if (DESCR(bean) == &descr_struct_CHUNKS)
return encode_chunk (g, bean);
if (DESCR(bean) == &descr_struct_CONTENTS)
return encode_content (g, bean);
if (DESCR(bean) == &descr_struct_CONTENTS_HEADERS)
return encode_header (g, bean);
if (DESCR(bean) == &descr_struct_ALIASES)
return encode_alias (g, bean);
if (DESCR(bean) == &descr_struct_PROPERTIES)
return encode_property (g, bean);
g_assert_not_reached ();
}
static int do_write_req(h2o_httpclient_t *_client, h2o_iovec_t chunk, int is_end_stream)
{
struct st_h2o_http1client_t *client = (struct st_h2o_http1client_t *)_client;
client->_body_buf_is_done = is_end_stream;
if (client->_body_buf == NULL)
h2o_buffer_init(&client->_body_buf, &h2o_socket_buffer_prototype);
if (chunk.len != 0) {
if (h2o_buffer_append(&client->_body_buf, chunk.base, chunk.len) == 0)
return -1;
}
if (client->sock->_cb.write != NULL)
return 0;
assert(client->_body_buf_in_flight == NULL || client->_body_buf_in_flight->size == 0);
swap_buffers(&client->_body_buf, &client->_body_buf_in_flight);
if (client->_body_buf_in_flight->size == 0) {
/* return immediately if the chunk is empty */
on_req_body_done(client->sock, NULL);
return 0;
}
h2o_iovec_t iov = h2o_iovec_init(client->_body_buf_in_flight->bytes, client->_body_buf_in_flight->size);
if (client->_is_chunked) {
h2o_iovec_t bufs[3];
size_t bufcnt = encode_chunk(client, bufs, iov);
h2o_socket_write(client->sock, bufs, bufcnt, on_req_body_done);
} else {
h2o_socket_write(client->sock, &iov, 1, on_req_body_done);
}
return 0;
}
void encode_file(char *file, int nativeBlockNum, int parityBlockNum)
{
int chunkSize = 1;
int totalSize;
FILE *fp_in;
FILE *fp_out;
if( ( fp_in = fopen(file,"rb") ) == NULL )
{
printf("Can not open source file!\n");
exit(0);
}
fseek(fp_in, 0L, SEEK_END);
//ftell() get the total size of the file
totalSize = ftell(fp_in);
chunkSize = (totalSize / nativeBlockNum) + ( totalSize%nativeBlockNum != 0 );
// chunkSize = (ftell(fp_in) / nativeBlockNum) + ( ftell(fp_in)%nativeBlockNum != 0 );
// chunkSize = (int) (ceil( (long double) (ftell(fp_in) / nativeBlockNum)) );
uint8_t *dataBuf; //host
uint8_t *codeBuf; //host
int dataSize = nativeBlockNum*chunkSize*sizeof(uint8_t);
int codeSize = parityBlockNum*chunkSize*sizeof(uint8_t);
dataBuf = (uint8_t*) malloc( nativeBlockNum*chunkSize*sizeof(uint8_t) );
memset(dataBuf, 0, dataSize);
codeBuf = (uint8_t*) malloc( parityBlockNum*chunkSize*sizeof(uint8_t) );
memset(codeBuf, 0, codeSize);
int i;
for(i=0; i<nativeBlockNum; i++)
{
if( fseek( fp_in, i*chunkSize, SEEK_SET ) == -1 )
{
printf("fseek error!\n");
exit(0);
}
if( fread( dataBuf+i*chunkSize, sizeof(uint8_t), chunkSize, fp_in ) == EOF )
{
printf("fread error!\n");
exit(0);
}
}
fclose(fp_in);
struct timespec start, end;
double totalTime;
clock_gettime(CLOCK_REALTIME,&start);
// // setup table for GF(2^8)
// setup_tables(8);
uint8_t *encodingMatrix;
encodingMatrix = (uint8_t*) malloc( parityBlockNum*nativeBlockNum*sizeof(uint8_t) );
gen_encoding_matrix(encodingMatrix, parityBlockNum, nativeBlockNum);
write_metadata(totalSize, parityBlockNum, nativeBlockNum, encodingMatrix);
encode_chunk(dataBuf, encodingMatrix, codeBuf, nativeBlockNum, parityBlockNum, chunkSize);
// matrix_mul(encodingMatrix, dataBuf, codeBuf, parityBlockNum, nativeBlockNum, chunkSize);
/*
//int i;
int j;
int k;
int n=parityBlockNum;
int p=nativeBlockNum;
int m=chunkSize;
for(i=0; i<n; i++)
{
for(j=0; j<m; j++)
{
codeBuf[i*m+j] = 0;
for(k=0; k<p; k++)
{
// C[i*m+j] = gf_add(C[i*m+j], gf_mul(A[i*p+k],B[k*m+j]));
codeBuf[i*m+j] ^= gf_mul(encodingMatrix[i*p+k],dataBuf[k*m+j]);
}
}
}
*/
clock_gettime(CLOCK_REALTIME,&end);
totalTime = (double)(end.tv_sec-start.tv_sec)*1000+(double)(end.tv_nsec-start.tv_nsec)/(double)1000000L;
printf("Total CPU encoding time: %fms\n", totalTime);
char output_file_name[100];
for(i=0; i<nativeBlockNum; i++)
{
sprintf(output_file_name, "_%d_", i);
strcat(output_file_name, file);
if( ( fp_out = fopen(output_file_name, "wb") ) == NULL )
{
printf("Can not open source file!\n");
exit(0);
}
if( fwrite(dataBuf+i*chunkSize, sizeof(uint8_t), chunkSize, fp_out ) != sizeof(uint8_t)*chunkSize )
{
printf("fwrite error!\n");
exit(0);
}
fclose(fp_out);
}
for(i=0; i<parityBlockNum; i++)
{
sprintf(output_file_name, "_%d_", i+nativeBlockNum);
strcat(output_file_name, file);
if( ( fp_out = fopen(output_file_name, "wb") ) == NULL )
{
printf("Can not open source file!\n");
exit(0);
}
if( fwrite(codeBuf+i*chunkSize, sizeof(uint8_t), chunkSize, fp_out ) != sizeof(uint8_t)*chunkSize )
{
printf("fwrite error!\n");
exit(0);
}
fclose(fp_out);
}
free(dataBuf);
free(codeBuf);
free(encodingMatrix);
}
int main(int argc, char *argv[])
{
int nativeBlockNum = 4;
int parityBlockNum = 2;
// int chunkSize = sizeof(uint8_t);
int chunkSize = 1;
// char[] file_name_src = argv[1];
FILE *fp_in;
FILE *fp_out;
int i;
if( ( fp_in = fopen(argv[1],"rb") ) == NULL )
{
printf("Can not open source file!\n");
exit(0);
}
fseek(fp_in, 0L, SEEK_END);
chunkSize = ftell(fp_in)/nativeBlockNum; //ftell() get the total size of the file
uint8_t *dataBuf;
uint8_t *codeBuf;
dataBuf = (unsigned char*) malloc( nativeBlockNum*chunkSize*sizeof(uint8_t) );
codeBuf = (unsigned char*) malloc( parityBlockNum*chunkSize*sizeof(uint8_t) );
for(i=0; i<nativeBlockNum; i++)
{
if( fseek(fp_in, i*chunkSize, SEEK_SET) == -1 )
{
printf("fseek error!\n");
exit(0);
}
if( fread(dataBuf+i*chunkSize, sizeof(uint8_t), chunkSize, fp_in ) != sizeof(uint8_t)*chunkSize )
{
printf("fread error!\n");
exit(0);
}
}
fclose(fp_in);
// setup table for GF(2^8)
setup_tables(8);
gen_encoding_matrix(parityBlockNum, nativeBlockNum);
encode_chunk(dataBuf, encodingMatrix, codeBuf, nativeBlockNum, parityBlockNum, chunkSize);
#ifdef DEBUG
show_code_chunk(codeBuf, parityBlockNum, chunkSize);
#endif
char output_file_name[10];
for(i=0; i<parityBlockNum; i++)
{
sprintf(output_file_name, "code_%d", i);
if( ( fp_out = fopen(output_file_name, "wb") ) == NULL )
{
printf("Can not open source file!\n");
exit(0);
}
if( fwrite(codeBuf+i*chunkSize, sizeof(uint8_t), chunkSize, fp_out ) != sizeof(uint8_t)*chunkSize )
{
printf("fwrite error!\n");
exit(0);
}
fclose(fp_out);
}
// uint8_t testData[4] = {9,1,2,0};
// uint8_t *testCode;
//
// // setup table for GF(2^8)
// setup_tables(8);
// gen_encoding_matrix(parityBlockNum, nativeBlockNum);
// testCode = (unsigned char*)malloc(parityBlockNum*chunkSize);
// encode_chunk(testData, encodingMatrix, testCode, nativeBlockNum, parityBlockNum, chunkSize);
//
//#ifdef DEBUG
//showgflog();
//printf("\n");
//showgfilog();
//printf("\n");
//#endif
// show_code_chunk(testCode, parityBlockNum, chunkSize);
//#ifdef DEBUG
//printf("\n");
//show_encoding_matrix(parityBlockNum, nativeBlockNum);
//#endif
free(encodingMatrix);
}
void ChunkEncoder::encode_chunk(const std::shared_ptr<Chunk>& chunk, const std::vector<DataType>& column_data_types,
const SegmentEncodingSpec& segment_encoding_spec) {
const auto chunk_encoding_spec = ChunkEncodingSpec{chunk->column_count(), segment_encoding_spec};
encode_chunk(chunk, column_data_types, chunk_encoding_spec);
}
static int update_pgsql(const char *database, const char *table, const char *keyfield,
const char *lookup, va_list ap)
{
PGresult *result = NULL;
int numrows = 0, pgerror;
char sql[256], escapebuf[2049], semibuf[1024];
const char *newparam, *newval;
if (!table) {
ast_log(LOG_WARNING, "Postgresql RealTime: No table specified.\n");
return -1;
}
/* Get the first parameter and first value in our list of passed paramater/value pairs */
newparam = va_arg(ap, const char *);
newval = va_arg(ap, const char *);
if (!newparam || !newval) {
ast_log(LOG_WARNING,
"Postgresql RealTime: Realtime retrieval requires at least 1 parameter and 1 value to search on.\n");
if (pgsqlConn) {
PQfinish(pgsqlConn);
pgsqlConn = NULL;
};
return -1;
}
/* Create the first part of the query using the first parameter/value pairs we just extracted
If there is only 1 set, then we have our query. Otherwise, loop thru the list and concat */
PQescapeStringConn(pgsqlConn, escapebuf, encode_chunk(newval, semibuf, sizeof(semibuf)), (sizeof(escapebuf) - 1) / 2, &pgerror);
if (pgerror) {
ast_log(LOG_ERROR, "Postgres detected invalid input: '%s'\n", newval);
va_end(ap);
return -1;
}
snprintf(sql, sizeof(sql), "UPDATE %s SET %s = '%s'", table, newparam, escapebuf);
while ((newparam = va_arg(ap, const char *))) {
newval = va_arg(ap, const char *);
PQescapeStringConn(pgsqlConn, escapebuf, encode_chunk(newval, semibuf, sizeof(semibuf)), (sizeof(escapebuf) - 1) / 2, &pgerror);
if (pgerror) {
ast_log(LOG_ERROR, "Postgres detected invalid input: '%s'\n", newval);
va_end(ap);
return -1;
}
snprintf(sql + strlen(sql), sizeof(sql) - strlen(sql), ", %s = '%s'", newparam,
escapebuf);
}
va_end(ap);
PQescapeStringConn(pgsqlConn, escapebuf, lookup, (sizeof(escapebuf) - 1) / 2, &pgerror);
if (pgerror) {
ast_log(LOG_ERROR, "Postgres detected invalid input: '%s'\n", lookup);
va_end(ap);
return -1;
}
snprintf(sql + strlen(sql), sizeof(sql) - strlen(sql), " WHERE %s = '%s'", keyfield,
escapebuf);
ast_log(LOG_DEBUG, "Postgresql RealTime: Update SQL: %s\n", sql);
/* We now have our complete statement; Lets connect to the server and execute it. */
ast_mutex_lock(&pgsql_lock);
if (!pgsql_reconnect(database)) {
ast_mutex_unlock(&pgsql_lock);
return -1;
}
if (!(result = PQexec(pgsqlConn, sql))) {
ast_log(LOG_WARNING,
"Postgresql RealTime: Failed to query database. Check debug for more info.\n");
ast_log(LOG_DEBUG, "Postgresql RealTime: Query: %s\n", sql);
ast_log(LOG_DEBUG, "Postgresql RealTime: Query Failed because: %s\n",
PQerrorMessage(pgsqlConn));
ast_mutex_unlock(&pgsql_lock);
return -1;
} else {
ExecStatusType result_status = PQresultStatus(result);
if (result_status != PGRES_COMMAND_OK
&& result_status != PGRES_TUPLES_OK
&& result_status != PGRES_NONFATAL_ERROR) {
ast_log(LOG_WARNING,
"Postgresql RealTime: Failed to query database. Check debug for more info.\n");
ast_log(LOG_DEBUG, "Postgresql RealTime: Query: %s\n", sql);
ast_log(LOG_DEBUG, "Postgresql RealTime: Query Failed because: %s (%s)\n",
PQresultErrorMessage(result), PQresStatus(result_status));
ast_mutex_unlock(&pgsql_lock);
return -1;
}
}
numrows = atoi(PQcmdTuples(result));
ast_mutex_unlock(&pgsql_lock);
ast_log(LOG_DEBUG, "Postgresql RealTime: Updated %d rows on table: %s\n", numrows,
table);
/* From http://dev.pgsql.com/doc/pgsql/en/pgsql-affected-rows.html
* An integer greater than zero indicates the number of rows affected
* Zero indicates that no records were updated
* -1 indicates that the query returned an error (although, if the query failed, it should have been caught above.)
*/
if (numrows >= 0)
return (int) numrows;
return -1;
}
static struct ast_config *realtime_multi_pgsql(const char *database, const char *table, va_list ap)
{
PGresult *result = NULL;
int num_rows = 0, pgerror;
char sql[256], escapebuf[2049], semibuf[1024];
const char *initfield = NULL;
char *stringp;
char *chunk;
char *op;
const char *newparam, *newval;
struct ast_realloca ra;
struct ast_variable *var = NULL;
struct ast_config *cfg = NULL;
struct ast_category *cat = NULL;
if (!table) {
ast_log(LOG_WARNING, "Postgresql RealTime: No table specified.\n");
return NULL;
}
memset(&ra, 0, sizeof(ra));
if (!(cfg = ast_config_new()))
return NULL;
/* Get the first parameter and first value in our list of passed paramater/value pairs */
newparam = va_arg(ap, const char *);
newval = va_arg(ap, const char *);
if (!newparam || !newval) {
ast_log(LOG_WARNING,
"Postgresql RealTime: Realtime retrieval requires at least 1 parameter and 1 value to search on.\n");
if (pgsqlConn) {
PQfinish(pgsqlConn);
pgsqlConn = NULL;
};
return NULL;
}
initfield = ast_strdupa(newparam);
if ((op = strchr(initfield, ' '))) {
*op = '\0';
}
/* Create the first part of the query using the first parameter/value pairs we just extracted
If there is only 1 set, then we have our query. Otherwise, loop thru the list and concat */
if (!strchr(newparam, ' '))
op = " =";
else
op = "";
PQescapeStringConn(pgsqlConn, escapebuf, encode_chunk(newval, semibuf, sizeof(semibuf)), (sizeof(escapebuf) - 1) / 2, &pgerror);
if (pgerror) {
ast_log(LOG_ERROR, "Postgres detected invalid input: '%s'\n", newval);
va_end(ap);
return NULL;
}
snprintf(sql, sizeof(sql), "SELECT * FROM %s WHERE %s%s '%s'", table, newparam, op,
escapebuf);
while ((newparam = va_arg(ap, const char *))) {
newval = va_arg(ap, const char *);
if (!strchr(newparam, ' '))
op = " =";
else
op = "";
PQescapeStringConn(pgsqlConn, escapebuf, encode_chunk(newval, semibuf, sizeof(semibuf)), (sizeof(escapebuf) - 1) / 2, &pgerror);
if (pgerror) {
ast_log(LOG_ERROR, "Postgres detected invalid input: '%s'\n", newval);
va_end(ap);
return NULL;
}
snprintf(sql + strlen(sql), sizeof(sql) - strlen(sql), " AND %s%s '%s'", newparam,
op, escapebuf);
}
if (initfield) {
snprintf(sql + strlen(sql), sizeof(sql) - strlen(sql), " ORDER BY %s", initfield);
}
va_end(ap);
/* We now have our complete statement; Lets connect to the server and execute it. */
ast_mutex_lock(&pgsql_lock);
if (!pgsql_reconnect(database)) {
ast_mutex_unlock(&pgsql_lock);
return NULL;
}
if (!(result = PQexec(pgsqlConn, sql))) {
ast_log(LOG_WARNING,
"Postgresql RealTime: Failed to query database. Check debug for more info.\n");
ast_log(LOG_DEBUG, "Postgresql RealTime: Query: %s\n", sql);
ast_log(LOG_DEBUG, "Postgresql RealTime: Query Failed because: %s\n",
PQerrorMessage(pgsqlConn));
ast_mutex_unlock(&pgsql_lock);
return NULL;
} else {
ExecStatusType result_status = PQresultStatus(result);
if (result_status != PGRES_COMMAND_OK
&& result_status != PGRES_TUPLES_OK
&& result_status != PGRES_NONFATAL_ERROR) {
ast_log(LOG_WARNING,
"Postgresql RealTime: Failed to query database. Check debug for more info.\n");
ast_log(LOG_DEBUG, "Postgresql RealTime: Query: %s\n", sql);
ast_log(LOG_DEBUG, "Postgresql RealTime: Query Failed because: %s (%s)\n",
PQresultErrorMessage(result), PQresStatus(result_status));
ast_mutex_unlock(&pgsql_lock);
return NULL;
}
}
ast_log(LOG_DEBUG, "2Postgresql RealTime: Result=%p Query: %s\n", result, sql);
if ((num_rows = PQntuples(result)) > 0) {
int numFields = PQnfields(result);
int i = 0;
int rowIndex = 0;
char **fieldnames = NULL;
ast_log(LOG_DEBUG, "Postgresql RealTime: Found %d rows.\n", num_rows);
if (!(fieldnames = ast_calloc(1, numFields * sizeof(char *)))) {
ast_mutex_unlock(&pgsql_lock);
PQclear(result);
return NULL;
}
for (i = 0; i < numFields; i++)
fieldnames[i] = PQfname(result, i);
for (rowIndex = 0; rowIndex < num_rows; rowIndex++) {
var = NULL;
if (!(cat = ast_category_new("")))
continue;
for (i = 0; i < numFields; i++) {
stringp = PQgetvalue(result, rowIndex, i);
while (stringp) {
chunk = strsep(&stringp, ";");
if (chunk && !ast_strlen_zero(decode_chunk(ast_strip(chunk)))) {
if (initfield && !strcmp(initfield, fieldnames[i])) {
ast_category_rename(cat, chunk);
}
var = ast_variable_new(fieldnames[i], chunk);
ast_variable_append(cat, var);
}
}
}
ast_category_append(cfg, cat);
}
ast_free(fieldnames);
} else {
ast_log(LOG_DEBUG,
"Postgresql RealTime: Could not find any rows in table %s.\n", table);
}
ast_mutex_unlock(&pgsql_lock);
PQclear(result);
return cfg;
}
