CAMLprim value PQsendQueryPrepared_stub(
value v_conn, value v_stm_name, value v_params, value v_binary_params)
{
PGconn *conn = get_conn(v_conn);
const char *stm_name = String_val(v_stm_name);
size_t nparams = Wosize_val(v_params);
const char * const *params = copy_params_shallow(v_params, nparams);
int *lengths, *formats, res;
copy_binary_params(v_params, v_binary_params, nparams, &formats, &lengths);
res =
PQsendQueryPrepared(conn, stm_name, nparams, params, lengths, formats, 0);
free_binary_params(formats, lengths);
free_params_shallow(params, nparams);
return Val_int(res);
}
static awk_value_t *
do_pg_sendqueryprepared(int nargs, awk_value_t *result)
{
PGconn *conn;
awk_value_t command;
int nParams;
const char **paramValues;
int res;
if (do_lint && (nargs > 4))
lintwarn(ext_id, _("pg_sendqueryprepared: called with too many arguments"));
if (!(conn = find_handle(conns, 0))) {
set_ERRNO(_("pg_sendqueryprepared called with unknown connection handle"));
RET_NUM(0);
}
if ((nParams = get_params(nargs, 2, ¶mValues)) < 0) {
set_ERRNO(_("pg_sendqueryprepared called with negative nParams"));
RET_NUM(0);
}
if (!get_argument(1, AWK_STRING, &command)) {
set_ERRNO(_("pg_sendqueryprepared 2nd argument should be a string"));
RET_NUM(0);
}
res = PQsendQueryPrepared(conn, command.str_value.str, nParams,
paramValues, NULL, NULL, 0);
if (paramValues)
free(paramValues);
if (!res)
/* connection is probably bad */
set_ERRNO(PQerrorMessage(conn));
RET_NUM(res);
}
bool EpollPostgresql::sendNextQuery()
{
const PreparedStatement &firstEntry=queriesList.front();
#if defined(CATCHCHALLENGER_DB_PREPAREDSTATEMENT)
int query_id=0;
if(firstEntry.id!=NULL)
query_id=PQsendQueryPrepared(conn,firstEntry.id,firstEntry.paramValuesCount,firstEntry.paramValues, NULL, NULL, 0);
else
query_id=PQsendQuery(conn,firstEntry.query.c_str());
#else
const int &query_id=PQsendQuery(conn,firstEntry.query.c_str());
#endif
if(query_id==0)
{
std::string tempString;
{
unsigned int index=0;
while(index<queriesList.size())
{
if(!tempString.empty())
tempString+=";";
tempString+=queriesList.at(index).query;
index++;
}
}
#ifdef DEBUG_MESSAGE_CLIENT_SQL
std::cerr << simplifiedstrCoPG << ", ";
#endif
std::cerr << "query async send failed: " << errorMessage() << ", where query list is not empty: " << tempString << std::endl;
return false;
}
#ifdef DEBUG_MESSAGE_CLIENT_SQL
std::cout << simplifiedstrCoPG << ", query " << firstEntry.query << " from queue at " << std::string(__FILE__) << ":" << std::to_string(__LINE__) << std::endl;
#endif
return true;
}
static int dbd_pgsql_pselect_internal(apr_pool_t *pool, apr_dbd_t *sql,
apr_dbd_results_t **results,
apr_dbd_prepared_t *statement,
int seek, const char **values,
const int *len, const int *fmt)
{
PGresult *res;
int rv;
int ret = 0;
if (seek) { /* synchronous query */
if (TXN_IGNORE_ERRORS(sql->trans)) {
PGresult *res = PQexec(sql->conn, "SAVEPOINT APR_DBD_TXN_SP");
if (res) {
int ret = PQresultStatus(res);
PQclear(res);
if (!dbd_pgsql_is_success(ret)) {
sql->trans->errnum = ret;
return PGRES_FATAL_ERROR;
}
} else {
sql->trans->errnum = ret;
return PGRES_FATAL_ERROR;
}
}
if (statement->prepared) {
res = PQexecPrepared(sql->conn, statement->name, statement->nargs,
values, len, fmt, 0);
}
else {
res = PQexecParams(sql->conn, statement->name, statement->nargs, 0,
values, len, fmt, 0);
}
if (res) {
ret = PQresultStatus(res);
if (dbd_pgsql_is_success(ret)) {
ret = 0;
}
else {
PQclear(res);
}
}
else {
ret = PGRES_FATAL_ERROR;
}
if (ret != 0) {
if (TXN_IGNORE_ERRORS(sql->trans)) {
PGresult *res = PQexec(sql->conn,
"ROLLBACK TO SAVEPOINT APR_DBD_TXN_SP");
if (res) {
int ret = PQresultStatus(res);
PQclear(res);
if (!dbd_pgsql_is_success(ret)) {
sql->trans->errnum = ret;
return PGRES_FATAL_ERROR;
}
} else {
sql->trans->errnum = ret;
return PGRES_FATAL_ERROR;
}
} else if (TXN_NOTICE_ERRORS(sql->trans)){
sql->trans->errnum = ret;
}
return ret;
} else {
if (TXN_IGNORE_ERRORS(sql->trans)) {
PGresult *res = PQexec(sql->conn,
"RELEASE SAVEPOINT APR_DBD_TXN_SP");
if (res) {
int ret = PQresultStatus(res);
PQclear(res);
if (!dbd_pgsql_is_success(ret)) {
sql->trans->errnum = ret;
return PGRES_FATAL_ERROR;
}
} else {
sql->trans->errnum = ret;
return PGRES_FATAL_ERROR;
}
}
}
if (!*results) {
*results = apr_pcalloc(pool, sizeof(apr_dbd_results_t));
}
(*results)->res = res;
(*results)->ntuples = PQntuples(res);
(*results)->sz = PQnfields(res);
(*results)->random = seek;
(*results)->pool = pool;
apr_pool_cleanup_register(pool, res, clear_result,
apr_pool_cleanup_null);
}
else {
if (TXN_IGNORE_ERRORS(sql->trans)) {
PGresult *res = PQexec(sql->conn, "SAVEPOINT APR_DBD_TXN_SP");
if (res) {
int ret = PQresultStatus(res);
PQclear(res);
if (!dbd_pgsql_is_success(ret)) {
sql->trans->errnum = ret;
return PGRES_FATAL_ERROR;
}
} else {
sql->trans->errnum = ret;
return PGRES_FATAL_ERROR;
}
}
if (statement->prepared) {
rv = PQsendQueryPrepared(sql->conn, statement->name,
statement->nargs, values, len, fmt, 0);
}
else {
rv = PQsendQueryParams(sql->conn, statement->name,
statement->nargs, 0, values, len, fmt, 0);
}
if (rv == 0) {
if (TXN_IGNORE_ERRORS(sql->trans)) {
PGresult *res = PQexec(sql->conn,
"ROLLBACK TO SAVEPOINT APR_DBD_TXN_SP");
if (res) {
int ret = PQresultStatus(res);
PQclear(res);
if (!dbd_pgsql_is_success(ret)) {
sql->trans->errnum = ret;
return PGRES_FATAL_ERROR;
}
} else {
sql->trans->errnum = ret;
return PGRES_FATAL_ERROR;
}
} else if (TXN_NOTICE_ERRORS(sql->trans)){
sql->trans->errnum = 1;
}
return 1;
} else {
if (TXN_IGNORE_ERRORS(sql->trans)) {
PGresult *res = PQexec(sql->conn,
"RELEASE SAVEPOINT APR_DBD_TXN_SP");
if (res) {
int ret = PQresultStatus(res);
PQclear(res);
if (!dbd_pgsql_is_success(ret)) {
sql->trans->errnum = ret;
return PGRES_FATAL_ERROR;
}
} else {
sql->trans->errnum = ret;
return PGRES_FATAL_ERROR;
}
}
}
if (!*results) {
*results = apr_pcalloc(pool, sizeof(apr_dbd_results_t));
}
(*results)->random = seek;
(*results)->handle = sql->conn;
(*results)->pool = pool;
}
return ret;
}
void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile)
{
struct index_thread_data * thread_data;
pthread_mutex_t count_mutex = PTHREAD_MUTEX_INITIALIZER;
int tuples, count, sleepcount;
time_t rankStartTime;
int rankTotalTuples;
int rankCountTuples;
float rankPerSecond;
PGconn *conn;
PGresult * res;
PGresult * resSectors;
PGresult * resPlaces;
PGresult * resNULL;
int rank;
int i;
int iSector;
int iResult;
const char *paramValues[2];
int paramLengths[2];
int paramFormats[2];
uint32_t paramRank;
uint32_t paramSector;
uint32_t sector;
xmlTextWriterPtr writer;
pthread_mutex_t writer_mutex = PTHREAD_MUTEX_INITIALIZER;
Oid pg_prepare_params[2];
conn = PQconnectdb(conninfo);
if (PQstatus(conn) != CONNECTION_OK)
{
fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
pg_prepare_params[0] = PG_OID_INT4;
res = PQprepare(conn, "index_sectors",
"select geometry_sector,count(*) from placex where rank_search = $1 and indexed_status > 0 group by geometry_sector order by geometry_sector",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed preparing index_sectors: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
PQclear(res);
pg_prepare_params[0] = PG_OID_INT4;
res = PQprepare(conn, "index_nosectors",
"select 0::integer,count(*) from placex where rank_search = $1 and indexed_status > 0",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed preparing index_sectors: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
PQclear(res);
pg_prepare_params[0] = PG_OID_INT4;
pg_prepare_params[1] = PG_OID_INT4;
res = PQprepare(conn, "index_sector_places",
"select place_id from placex where rank_search = $1 and geometry_sector = $2 and indexed_status > 0",
2, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed preparing index_sector_places: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
PQclear(res);
pg_prepare_params[0] = PG_OID_INT4;
res = PQprepare(conn, "index_nosector_places",
"select place_id from placex where rank_search = $1 and indexed_status > 0 order by geometry_sector",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed preparing index_nosector_places: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
PQclear(res);
// Build the data for each thread
thread_data = (struct index_thread_data *)malloc(sizeof(struct index_thread_data)*num_threads);
for (i = 0; i < num_threads; i++)
{
thread_data[i].conn = PQconnectdb(conninfo);
if (PQstatus(thread_data[i].conn) != CONNECTION_OK)
{
fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(thread_data[i].conn));
exit(EXIT_FAILURE);
}
pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(thread_data[i].conn, "index_placex",
"update placex set indexed_status = 0 where place_id = $1",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed preparing index_placex: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
PQclear(res);
/*res = PQexec(thread_data[i].conn, "set enable_seqscan = false");
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed disabling sequential scan: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
PQclear(res);*/
nominatim_exportCreatePreparedQueries(thread_data[i].conn);
}
// Create the output file
writer = NULL;
if (structuredoutputfile)
{
writer = nominatim_exportXMLStart(structuredoutputfile);
}
fprintf(stderr, "Starting indexing rank (%i to %i) using %i treads\n", rank_min, rank_max, num_threads);
for (rank = rank_min; rank <= rank_max; rank++)
{
fprintf(stderr, "Starting rank %d\n", rank);
rankCountTuples = 0;
rankPerSecond = 0;
paramRank = PGint32(rank);
paramValues[0] = (char *)¶mRank;
paramLengths[0] = sizeof(paramRank);
paramFormats[0] = 1;
// if (rank < 16)
// resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
// else
resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
{
fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resSectors);
exit(EXIT_FAILURE);
}
if (PQftype(resSectors, 0) != PG_OID_INT4)
{
fprintf(stderr, "Sector value has unexpected type\n");
PQclear(resSectors);
exit(EXIT_FAILURE);
}
if (PQftype(resSectors, 1) != PG_OID_INT8)
{
fprintf(stderr, "Sector value has unexpected type\n");
PQclear(resSectors);
exit(EXIT_FAILURE);
}
rankTotalTuples = 0;
for (iSector = 0; iSector < PQntuples(resSectors); iSector++)
{
rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)));
}
rankStartTime = time(0);
for (iSector = 0; iSector <= PQntuples(resSectors); iSector++)
{
if (iSector > 0)
{
resPlaces = PQgetResult(conn);
if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
{
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resPlaces);
exit(EXIT_FAILURE);
}
if (PQftype(resPlaces, 0) != PG_OID_INT8)
{
fprintf(stderr, "Place_id value has unexpected type\n");
PQclear(resPlaces);
exit(EXIT_FAILURE);
}
resNULL = PQgetResult(conn);
if (resNULL != NULL)
{
fprintf(stderr, "Unexpected non-null response\n");
exit(EXIT_FAILURE);
}
}
if (iSector < PQntuples(resSectors))
{
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
// fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
// Get all the place_id's for this sector
paramRank = PGint32(rank);
paramValues[0] = (char *)¶mRank;
paramLengths[0] = sizeof(paramRank);
paramFormats[0] = 1;
paramSector = PGint32(sector);
paramValues[1] = (char *)¶mSector;
paramLengths[1] = sizeof(paramSector);
paramFormats[1] = 1;
if (rankTotalTuples-rankCountTuples < num_threads*1000)
{
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
}
else
{
iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
}
if (!iResult)
{
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resPlaces);
exit(EXIT_FAILURE);
}
}
if (iSector > 0)
{
count = 0;
rankPerSecond = 0;
tuples = PQntuples(resPlaces);
if (tuples > 0)
{
// Spawn threads
for (i = 0; i < num_threads; i++)
{
thread_data[i].res = resPlaces;
thread_data[i].tuples = tuples;
thread_data[i].count = &count;
thread_data[i].count_mutex = &count_mutex;
thread_data[i].writer = writer;
thread_data[i].writer_mutex = &writer_mutex;
pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]);
}
// Monitor threads to give user feedback
sleepcount = 0;
while (count < tuples)
{
usleep(1000);
// Aim for one update per second
if (sleepcount++ > 2000)
{
rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
fprintf(stderr, " Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond);
sleepcount = 0;
}
}
// Wait for everything to finish
for (i = 0; i < num_threads; i++)
{
pthread_join(thread_data[i].thread, NULL);
}
rankCountTuples += tuples;
}
// Finished sector
rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);
fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
PQclear(resPlaces);
}
if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors))
{
iSector = PQntuples(resSectors) - 1;
}
}
// Finished rank
fprintf(stderr, "\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
PQclear(resSectors);
}
if (writer)
{
nominatim_exportXMLEnd(writer);
}
}
bool EpollPostgresql::asyncPreparedWrite(const std::string &query,char * const id,const std::vector<std::string> &values)
{
if(conn==NULL)
{
std::cerr << "pg not connected" << std::endl;
return false;
}
size_t bufferSize=0;
const char *paramValues[values.size()];
{
uint8_t index=0;
while(index<values.size())
{
paramValues[index]=values.at(index).c_str();
bufferSize++;
bufferSize+=values.at(index).size();
index++;
}
}
PreparedStatement tempQuery;
tempQuery.id=id;
tempQuery.paramValuesCount=static_cast<uint8_t>(values.size());
tempQuery.paramValuesBuffer=NULL;
tempQuery.query=query;
if(queue.size()>0 || result!=NULL)
{
tempQuery.paramValuesBuffer=new char[bufferSize+1];
tempQuery.paramValues=(char **)malloc(sizeof(char *)*values.size());
{
size_t cursor=0;
uint8_t index=0;
while(index<values.size())
{
tempQuery.paramValues[index]=tempQuery.paramValuesBuffer+cursor;
strcpy(tempQuery.paramValuesBuffer+cursor,values.at(index).c_str());
cursor+=values.at(index).size()+1;
index++;
}
}
queue.push_back(emptyCallback);
queriesList.push_back(tempQuery);
return true;
}
start = std::chrono::high_resolution_clock::now();
queriesList.push_back(tempQuery);
#ifdef CATCHCHALLENGER_EXTRA_CHECK
const int &stringlen=static_cast<int>(query.size());
if(stringlen==0)
{
std::cerr << "query " << query << ", stringlen==0" << std::endl;
abort();
}
#endif
#ifdef DEBUG_MESSAGE_CLIENT_SQL
std::cout << simplifiedstrCoPG << ", query " << CatchChallenger::SqlFunction::replaceSQLValues(query,values) << " at " << std::string(__FILE__) << ":" << std::to_string(__LINE__) << std::endl;
#endif
const int &query_id=PQsendQueryPrepared(conn,id,static_cast<int>(values.size()),paramValues, NULL, NULL, 0);
if(query_id==0)
{
std::cerr << "query send failed" << std::endl;
return false;
}
queue.push_back(emptyCallback);
return true;
}
Result
PreparedStatement::execute()
{
PGconn *conn = m_conn.lock().get();
boost::shared_ptr<PGresult> result, next;
int nParams = (int)m_params.size();
Oid *paramTypes = NULL;
int *paramLengths = NULL, *paramFormats = NULL;
const char **params = NULL;
if (nParams) {
if (m_name.empty())
paramTypes = &m_paramTypes[0];
params = &m_params[0];
paramLengths = &m_paramLengths[0];
paramFormats = &m_paramFormats[0];
}
const char *api = NULL;
#ifndef WINDOWS
SchedulerSwitcher switcher(m_scheduler);
#endif
if (m_name.empty()) {
#ifndef WINDOWS
if (m_scheduler) {
api = "PQsendQueryParams";
if (!PQsendQueryParams(conn, m_command.c_str(),
nParams, paramTypes, params, paramLengths, paramFormats, m_resultFormat))
throwException(conn);
flush(conn, m_scheduler);
next.reset(nextResult(conn, m_scheduler), &PQclear);
while (next) {
result = next;
next.reset(nextResult(conn, m_scheduler), &PQclear);
if (next) {
ExecStatusType status = PQresultStatus(next.get());
MORDOR_LOG_VERBOSE(g_log) << conn << "PQresultStatus(" <<
next.get() << "): " << PQresStatus(status);
switch (status) {
case PGRES_COMMAND_OK:
case PGRES_TUPLES_OK:
break;
default:
throwException(next.get());
MORDOR_NOTREACHED();
}
}
}
} else
#endif
{
api = "PQexecParams";
result.reset(PQexecParams(conn, m_command.c_str(),
nParams, paramTypes, params, paramLengths, paramFormats, m_resultFormat),
&PQclear);
}
} else {
#ifndef WINDOWS
if (m_scheduler) {
api = "PQsendQueryPrepared";
if (!PQsendQueryPrepared(conn, m_name.c_str(),
nParams, params, paramLengths, paramFormats, 1))
throwException(conn);
flush(conn, m_scheduler);
next.reset(nextResult(conn, m_scheduler), &PQclear);
while (next) {
result = next;
next.reset(nextResult(conn, m_scheduler), &PQclear);
if (next) {
ExecStatusType status = PQresultStatus(next.get());
MORDOR_LOG_VERBOSE(g_log) << conn << "PQresultStatus(" <<
next.get() << "): " << PQresStatus(status);
switch (status) {
case PGRES_COMMAND_OK:
case PGRES_TUPLES_OK:
break;
default:
throwException(next.get());
MORDOR_NOTREACHED();
}
}
}
} else
#endif
{
api = "PQexecPrepared";
result.reset(PQexecPrepared(conn, m_name.c_str(),
nParams, params, paramLengths, paramFormats, m_resultFormat),
&PQclear);
}
}
if (!result)
throwException(conn);
ExecStatusType status = PQresultStatus(result.get());
MORDOR_ASSERT(api);
MORDOR_LOG_VERBOSE(g_log) << conn << " " << api << "(\"" << m_command
<< m_name << "\", " << nParams << "), PQresultStatus(" << result.get()
<< "): " << PQresStatus(status);
switch (status) {
case PGRES_COMMAND_OK:
case PGRES_TUPLES_OK:
return Result(result);
default:
throwException(result.get());
MORDOR_NOTREACHED();
}
}
void run_indexing(int rank, int interpolation, PGconn *conn, int num_threads,
struct index_thread_data * thread_data, const char *structuredoutputfile)
{
int tuples, count, sleepcount;
pthread_mutex_t count_mutex = PTHREAD_MUTEX_INITIALIZER;
time_t rankStartTime;
int rankTotalTuples;
int rankCountTuples;
float rankPerSecond;
PGresult * resSectors;
PGresult * resPlaces;
PGresult * resNULL;
int i;
int iSector;
int iResult;
const char *paramValues[2];
int paramLengths[2];
int paramFormats[2];
uint32_t paramRank;
uint32_t paramSector;
uint32_t sector;
xmlTextWriterPtr writer;
pthread_mutex_t writer_mutex = PTHREAD_MUTEX_INITIALIZER;
// Create the output file
writer = NULL;
if (structuredoutputfile)
{
writer = nominatim_exportXMLStart(structuredoutputfile);
}
if (interpolation)
{
fprintf(stderr, "Starting interpolation lines (location_property_osmline)\n");
}
else
{
fprintf(stderr, "Starting rank %d\n", rank);
}
rankCountTuples = 0;
rankPerSecond = 0;
paramRank = PGint32(rank);
paramValues[0] = (char *)¶mRank;
paramLengths[0] = sizeof(paramRank);
paramFormats[0] = 1;
if (interpolation)
{
resSectors = PQexecPrepared(conn, "index_sectors_osmline", 0, NULL, 0, NULL, 1);
}
else
{
resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
}
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
{
fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resSectors);
exit(EXIT_FAILURE);
}
if (PQftype(resSectors, 0) != PG_OID_INT4)
{
fprintf(stderr, "Sector value has unexpected type\n");
PQclear(resSectors);
exit(EXIT_FAILURE);
}
if (PQftype(resSectors, 1) != PG_OID_INT8)
{
fprintf(stderr, "Sector value has unexpected type\n");
PQclear(resSectors);
exit(EXIT_FAILURE);
}
rankTotalTuples = 0;
for (iSector = 0; iSector < PQntuples(resSectors); iSector++)
{
rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)));
}
rankStartTime = time(0);
for (iSector = 0; iSector <= PQntuples(resSectors); iSector++)
{
if (iSector > 0)
{
resPlaces = PQgetResult(conn);
if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
{
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resPlaces);
exit(EXIT_FAILURE);
}
if (PQftype(resPlaces, 0) != PG_OID_INT8)
{
fprintf(stderr, "Place_id value has unexpected type\n");
PQclear(resPlaces);
exit(EXIT_FAILURE);
}
resNULL = PQgetResult(conn);
if (resNULL != NULL)
{
fprintf(stderr, "Unexpected non-null response\n");
exit(EXIT_FAILURE);
}
}
if (iSector < PQntuples(resSectors))
{
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
// fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
// Get all the place_id's for this sector
paramRank = PGint32(rank);
paramSector = PGint32(sector);
if (rankTotalTuples-rankCountTuples < num_threads*1000)
{
// no sectors
if (interpolation)
{
iResult = PQsendQueryPrepared(conn, "index_nosector_places_osmline", 0, NULL, 0, NULL, 1);
}
else
{
paramValues[0] = (char *)¶mRank;
paramLengths[0] = sizeof(paramRank);
paramFormats[0] = 1;
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
}
}
else
{
if (interpolation)
{
iResult = PQsendQueryPrepared(conn, "index_sector_places_osmline", 1, paramValues, paramLengths, paramFormats, 1);
paramValues[0] = (char *)¶mSector;
paramLengths[0] = sizeof(paramSector);
paramFormats[0] = 1;
}
else
{
paramValues[0] = (char *)¶mRank;
paramLengths[0] = sizeof(paramRank);
paramFormats[0] = 1;
paramValues[1] = (char *)¶mSector;
paramLengths[1] = sizeof(paramSector);
paramFormats[1] = 1;
iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
}
}
if (!iResult)
{
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resPlaces);
exit(EXIT_FAILURE);
}
}
if (iSector > 0)
{
count = 0;
rankPerSecond = 0;
tuples = PQntuples(resPlaces);
if (tuples > 0)
{
// Spawn threads
for (i = 0; i < num_threads; i++)
{
thread_data[i].res = resPlaces;
thread_data[i].tuples = tuples;
thread_data[i].count = &count;
thread_data[i].count_mutex = &count_mutex;
thread_data[i].writer = writer;
thread_data[i].writer_mutex = &writer_mutex;
if (interpolation)
{
thread_data[i].table = 0; // use interpolations table
}
else
{
thread_data[i].table = 1; // use placex table
}
pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]);
}
// Monitor threads to give user feedback
sleepcount = 0;
while (count < tuples)
{
usleep(1000);
// Aim for one update per second
if (sleepcount++ > 1000)
{
rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
if(interpolation)
{
fprintf(stderr, " Done %i in %i @ %f per second - Interpolation lines ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond);
}
else
{
fprintf(stderr, " Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond);
}
sleepcount = 0;
}
}
// Wait for everything to finish
for (i = 0; i < num_threads; i++)
{
pthread_join(thread_data[i].thread, NULL);
}
rankCountTuples += tuples;
}
// Finished sector
rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);
fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
PQclear(resPlaces);
}
if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors))
{
iSector = PQntuples(resSectors) - 1;
}
}
// Finished rank
fprintf(stderr, "\r Done %i in %i @ %f per second - FINISHED\n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
PQclear(resSectors);
}
