CAMLprim value PQsendQueryPrepared_stub( value v_conn, value v_stm_name, value v_params, value v_binary_params) { PGconn *conn = get_conn(v_conn); const char *stm_name = String_val(v_stm_name); size_t nparams = Wosize_val(v_params); const char * const *params = copy_params_shallow(v_params, nparams); int *lengths, *formats, res; copy_binary_params(v_params, v_binary_params, nparams, &formats, &lengths); res = PQsendQueryPrepared(conn, stm_name, nparams, params, lengths, formats, 0); free_binary_params(formats, lengths); free_params_shallow(params, nparams); return Val_int(res); }
static awk_value_t * do_pg_sendqueryprepared(int nargs, awk_value_t *result) { PGconn *conn; awk_value_t command; int nParams; const char **paramValues; int res; if (do_lint && (nargs > 4)) lintwarn(ext_id, _("pg_sendqueryprepared: called with too many arguments")); if (!(conn = find_handle(conns, 0))) { set_ERRNO(_("pg_sendqueryprepared called with unknown connection handle")); RET_NUM(0); } if ((nParams = get_params(nargs, 2, ¶mValues)) < 0) { set_ERRNO(_("pg_sendqueryprepared called with negative nParams")); RET_NUM(0); } if (!get_argument(1, AWK_STRING, &command)) { set_ERRNO(_("pg_sendqueryprepared 2nd argument should be a string")); RET_NUM(0); } res = PQsendQueryPrepared(conn, command.str_value.str, nParams, paramValues, NULL, NULL, 0); if (paramValues) free(paramValues); if (!res) /* connection is probably bad */ set_ERRNO(PQerrorMessage(conn)); RET_NUM(res); }
bool EpollPostgresql::sendNextQuery() { const PreparedStatement &firstEntry=queriesList.front(); #if defined(CATCHCHALLENGER_DB_PREPAREDSTATEMENT) int query_id=0; if(firstEntry.id!=NULL) query_id=PQsendQueryPrepared(conn,firstEntry.id,firstEntry.paramValuesCount,firstEntry.paramValues, NULL, NULL, 0); else query_id=PQsendQuery(conn,firstEntry.query.c_str()); #else const int &query_id=PQsendQuery(conn,firstEntry.query.c_str()); #endif if(query_id==0) { std::string tempString; { unsigned int index=0; while(index<queriesList.size()) { if(!tempString.empty()) tempString+=";"; tempString+=queriesList.at(index).query; index++; } } #ifdef DEBUG_MESSAGE_CLIENT_SQL std::cerr << simplifiedstrCoPG << ", "; #endif std::cerr << "query async send failed: " << errorMessage() << ", where query list is not empty: " << tempString << std::endl; return false; } #ifdef DEBUG_MESSAGE_CLIENT_SQL std::cout << simplifiedstrCoPG << ", query " << firstEntry.query << " from queue at " << std::string(__FILE__) << ":" << std::to_string(__LINE__) << std::endl; #endif return true; }
static int dbd_pgsql_pselect_internal(apr_pool_t *pool, apr_dbd_t *sql, apr_dbd_results_t **results, apr_dbd_prepared_t *statement, int seek, const char **values, const int *len, const int *fmt) { PGresult *res; int rv; int ret = 0; if (seek) { /* synchronous query */ if (TXN_IGNORE_ERRORS(sql->trans)) { PGresult *res = PQexec(sql->conn, "SAVEPOINT APR_DBD_TXN_SP"); if (res) { int ret = PQresultStatus(res); PQclear(res); if (!dbd_pgsql_is_success(ret)) { sql->trans->errnum = ret; return PGRES_FATAL_ERROR; } } else { sql->trans->errnum = ret; return PGRES_FATAL_ERROR; } } if (statement->prepared) { res = PQexecPrepared(sql->conn, statement->name, statement->nargs, values, len, fmt, 0); } else { res = PQexecParams(sql->conn, statement->name, statement->nargs, 0, values, len, fmt, 0); } if (res) { ret = PQresultStatus(res); if (dbd_pgsql_is_success(ret)) { ret = 0; } else { PQclear(res); } } else { ret = PGRES_FATAL_ERROR; } if (ret != 0) { if (TXN_IGNORE_ERRORS(sql->trans)) { PGresult *res = PQexec(sql->conn, "ROLLBACK TO SAVEPOINT APR_DBD_TXN_SP"); if (res) { int ret = PQresultStatus(res); PQclear(res); if (!dbd_pgsql_is_success(ret)) { sql->trans->errnum = ret; return PGRES_FATAL_ERROR; } } else { sql->trans->errnum = ret; return PGRES_FATAL_ERROR; } } else if (TXN_NOTICE_ERRORS(sql->trans)){ sql->trans->errnum = ret; } return ret; } else { if (TXN_IGNORE_ERRORS(sql->trans)) { PGresult *res = PQexec(sql->conn, "RELEASE SAVEPOINT APR_DBD_TXN_SP"); if (res) { int ret = PQresultStatus(res); PQclear(res); if (!dbd_pgsql_is_success(ret)) { sql->trans->errnum = ret; return PGRES_FATAL_ERROR; } } else { sql->trans->errnum = ret; return PGRES_FATAL_ERROR; } } } if (!*results) { *results = apr_pcalloc(pool, sizeof(apr_dbd_results_t)); } (*results)->res = res; (*results)->ntuples = PQntuples(res); (*results)->sz = PQnfields(res); (*results)->random = seek; (*results)->pool = pool; apr_pool_cleanup_register(pool, res, clear_result, apr_pool_cleanup_null); } else { if (TXN_IGNORE_ERRORS(sql->trans)) { PGresult *res = PQexec(sql->conn, "SAVEPOINT APR_DBD_TXN_SP"); if (res) { int ret = PQresultStatus(res); PQclear(res); if (!dbd_pgsql_is_success(ret)) { sql->trans->errnum = ret; return PGRES_FATAL_ERROR; } } else { sql->trans->errnum = ret; return PGRES_FATAL_ERROR; } } if (statement->prepared) { rv = PQsendQueryPrepared(sql->conn, statement->name, statement->nargs, values, len, fmt, 0); } else { rv = PQsendQueryParams(sql->conn, statement->name, statement->nargs, 0, values, len, fmt, 0); } if (rv == 0) { if (TXN_IGNORE_ERRORS(sql->trans)) { PGresult *res = PQexec(sql->conn, "ROLLBACK TO SAVEPOINT APR_DBD_TXN_SP"); if (res) { int ret = PQresultStatus(res); PQclear(res); if (!dbd_pgsql_is_success(ret)) { sql->trans->errnum = ret; return PGRES_FATAL_ERROR; } } else { sql->trans->errnum = ret; return PGRES_FATAL_ERROR; } } else if (TXN_NOTICE_ERRORS(sql->trans)){ sql->trans->errnum = 1; } return 1; } else { if (TXN_IGNORE_ERRORS(sql->trans)) { PGresult *res = PQexec(sql->conn, "RELEASE SAVEPOINT APR_DBD_TXN_SP"); if (res) { int ret = PQresultStatus(res); PQclear(res); if (!dbd_pgsql_is_success(ret)) { sql->trans->errnum = ret; return PGRES_FATAL_ERROR; } } else { sql->trans->errnum = ret; return PGRES_FATAL_ERROR; } } } if (!*results) { *results = apr_pcalloc(pool, sizeof(apr_dbd_results_t)); } (*results)->random = seek; (*results)->handle = sql->conn; (*results)->pool = pool; } return ret; }
void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile) { struct index_thread_data * thread_data; pthread_mutex_t count_mutex = PTHREAD_MUTEX_INITIALIZER; int tuples, count, sleepcount; time_t rankStartTime; int rankTotalTuples; int rankCountTuples; float rankPerSecond; PGconn *conn; PGresult * res; PGresult * resSectors; PGresult * resPlaces; PGresult * resNULL; int rank; int i; int iSector; int iResult; const char *paramValues[2]; int paramLengths[2]; int paramFormats[2]; uint32_t paramRank; uint32_t paramSector; uint32_t sector; xmlTextWriterPtr writer; pthread_mutex_t writer_mutex = PTHREAD_MUTEX_INITIALIZER; Oid pg_prepare_params[2]; conn = PQconnectdb(conninfo); if (PQstatus(conn) != CONNECTION_OK) { fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn)); exit(EXIT_FAILURE); } pg_prepare_params[0] = PG_OID_INT4; res = PQprepare(conn, "index_sectors", "select geometry_sector,count(*) from placex where rank_search = $1 and indexed_status > 0 group by geometry_sector order by geometry_sector", 1, pg_prepare_params); if (PQresultStatus(res) != PGRES_COMMAND_OK) { fprintf(stderr, "Failed preparing index_sectors: %s\n", PQerrorMessage(conn)); exit(EXIT_FAILURE); } PQclear(res); pg_prepare_params[0] = PG_OID_INT4; res = PQprepare(conn, "index_nosectors", "select 0::integer,count(*) from placex where rank_search = $1 and indexed_status > 0", 1, pg_prepare_params); if (PQresultStatus(res) != PGRES_COMMAND_OK) { fprintf(stderr, "Failed preparing index_sectors: %s\n", PQerrorMessage(conn)); exit(EXIT_FAILURE); } PQclear(res); pg_prepare_params[0] = PG_OID_INT4; pg_prepare_params[1] = PG_OID_INT4; res = PQprepare(conn, "index_sector_places", "select place_id from placex where rank_search = $1 and geometry_sector = $2 and indexed_status > 0", 2, pg_prepare_params); if (PQresultStatus(res) != PGRES_COMMAND_OK) { fprintf(stderr, "Failed preparing index_sector_places: %s\n", PQerrorMessage(conn)); exit(EXIT_FAILURE); } PQclear(res); pg_prepare_params[0] = PG_OID_INT4; res = PQprepare(conn, "index_nosector_places", "select place_id from placex where rank_search = $1 and indexed_status > 0 order by geometry_sector", 1, pg_prepare_params); if (PQresultStatus(res) != PGRES_COMMAND_OK) { fprintf(stderr, "Failed preparing index_nosector_places: %s\n", PQerrorMessage(conn)); exit(EXIT_FAILURE); } PQclear(res); // Build the data for each thread thread_data = (struct index_thread_data *)malloc(sizeof(struct index_thread_data)*num_threads); for (i = 0; i < num_threads; i++) { thread_data[i].conn = PQconnectdb(conninfo); if (PQstatus(thread_data[i].conn) != CONNECTION_OK) { fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(thread_data[i].conn)); exit(EXIT_FAILURE); } pg_prepare_params[0] = PG_OID_INT8; res = PQprepare(thread_data[i].conn, "index_placex", "update placex set indexed_status = 0 where place_id = $1", 1, pg_prepare_params); if (PQresultStatus(res) != PGRES_COMMAND_OK) { fprintf(stderr, "Failed preparing index_placex: %s\n", PQerrorMessage(conn)); exit(EXIT_FAILURE); } PQclear(res); /*res = PQexec(thread_data[i].conn, "set enable_seqscan = false"); if (PQresultStatus(res) != PGRES_COMMAND_OK) { fprintf(stderr, "Failed disabling sequential scan: %s\n", PQerrorMessage(conn)); exit(EXIT_FAILURE); } PQclear(res);*/ nominatim_exportCreatePreparedQueries(thread_data[i].conn); } // Create the output file writer = NULL; if (structuredoutputfile) { writer = nominatim_exportXMLStart(structuredoutputfile); } fprintf(stderr, "Starting indexing rank (%i to %i) using %i treads\n", rank_min, rank_max, num_threads); for (rank = rank_min; rank <= rank_max; rank++) { fprintf(stderr, "Starting rank %d\n", rank); rankCountTuples = 0; rankPerSecond = 0; paramRank = PGint32(rank); paramValues[0] = (char *)¶mRank; paramLengths[0] = sizeof(paramRank); paramFormats[0] = 1; // if (rank < 16) // resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1); // else resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1); if (PQresultStatus(resSectors) != PGRES_TUPLES_OK) { fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn)); PQclear(resSectors); exit(EXIT_FAILURE); } if (PQftype(resSectors, 0) != PG_OID_INT4) { fprintf(stderr, "Sector value has unexpected type\n"); PQclear(resSectors); exit(EXIT_FAILURE); } if (PQftype(resSectors, 1) != PG_OID_INT8) { fprintf(stderr, "Sector value has unexpected type\n"); PQclear(resSectors); exit(EXIT_FAILURE); } rankTotalTuples = 0; for (iSector = 0; iSector < PQntuples(resSectors); iSector++) { rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))); } rankStartTime = time(0); for (iSector = 0; iSector <= PQntuples(resSectors); iSector++) { if (iSector > 0) { resPlaces = PQgetResult(conn); if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK) { fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn)); PQclear(resPlaces); exit(EXIT_FAILURE); } if (PQftype(resPlaces, 0) != PG_OID_INT8) { fprintf(stderr, "Place_id value has unexpected type\n"); PQclear(resPlaces); exit(EXIT_FAILURE); } resNULL = PQgetResult(conn); if (resNULL != NULL) { fprintf(stderr, "Unexpected non-null response\n"); exit(EXIT_FAILURE); } } if (iSector < PQntuples(resSectors)) { sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0))); // fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)))); // Get all the place_id's for this sector paramRank = PGint32(rank); paramValues[0] = (char *)¶mRank; paramLengths[0] = sizeof(paramRank); paramFormats[0] = 1; paramSector = PGint32(sector); paramValues[1] = (char *)¶mSector; paramLengths[1] = sizeof(paramSector); paramFormats[1] = 1; if (rankTotalTuples-rankCountTuples < num_threads*1000) { iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1); } else { iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1); } if (!iResult) { fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn)); PQclear(resPlaces); exit(EXIT_FAILURE); } } if (iSector > 0) { count = 0; rankPerSecond = 0; tuples = PQntuples(resPlaces); if (tuples > 0) { // Spawn threads for (i = 0; i < num_threads; i++) { thread_data[i].res = resPlaces; thread_data[i].tuples = tuples; thread_data[i].count = &count; thread_data[i].count_mutex = &count_mutex; thread_data[i].writer = writer; thread_data[i].writer_mutex = &writer_mutex; pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]); } // Monitor threads to give user feedback sleepcount = 0; while (count < tuples) { usleep(1000); // Aim for one update per second if (sleepcount++ > 2000) { rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1); fprintf(stderr, " Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond); sleepcount = 0; } } // Wait for everything to finish for (i = 0; i < num_threads; i++) { pthread_join(thread_data[i].thread, NULL); } rankCountTuples += tuples; } // Finished sector rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1); fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond); PQclear(resPlaces); } if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors)) { iSector = PQntuples(resSectors) - 1; } } // Finished rank fprintf(stderr, "\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond); PQclear(resSectors); } if (writer) { nominatim_exportXMLEnd(writer); } }
bool EpollPostgresql::asyncPreparedWrite(const std::string &query,char * const id,const std::vector<std::string> &values) { if(conn==NULL) { std::cerr << "pg not connected" << std::endl; return false; } size_t bufferSize=0; const char *paramValues[values.size()]; { uint8_t index=0; while(index<values.size()) { paramValues[index]=values.at(index).c_str(); bufferSize++; bufferSize+=values.at(index).size(); index++; } } PreparedStatement tempQuery; tempQuery.id=id; tempQuery.paramValuesCount=static_cast<uint8_t>(values.size()); tempQuery.paramValuesBuffer=NULL; tempQuery.query=query; if(queue.size()>0 || result!=NULL) { tempQuery.paramValuesBuffer=new char[bufferSize+1]; tempQuery.paramValues=(char **)malloc(sizeof(char *)*values.size()); { size_t cursor=0; uint8_t index=0; while(index<values.size()) { tempQuery.paramValues[index]=tempQuery.paramValuesBuffer+cursor; strcpy(tempQuery.paramValuesBuffer+cursor,values.at(index).c_str()); cursor+=values.at(index).size()+1; index++; } } queue.push_back(emptyCallback); queriesList.push_back(tempQuery); return true; } start = std::chrono::high_resolution_clock::now(); queriesList.push_back(tempQuery); #ifdef CATCHCHALLENGER_EXTRA_CHECK const int &stringlen=static_cast<int>(query.size()); if(stringlen==0) { std::cerr << "query " << query << ", stringlen==0" << std::endl; abort(); } #endif #ifdef DEBUG_MESSAGE_CLIENT_SQL std::cout << simplifiedstrCoPG << ", query " << CatchChallenger::SqlFunction::replaceSQLValues(query,values) << " at " << std::string(__FILE__) << ":" << std::to_string(__LINE__) << std::endl; #endif const int &query_id=PQsendQueryPrepared(conn,id,static_cast<int>(values.size()),paramValues, NULL, NULL, 0); if(query_id==0) { std::cerr << "query send failed" << std::endl; return false; } queue.push_back(emptyCallback); return true; }
Result PreparedStatement::execute() { PGconn *conn = m_conn.lock().get(); boost::shared_ptr<PGresult> result, next; int nParams = (int)m_params.size(); Oid *paramTypes = NULL; int *paramLengths = NULL, *paramFormats = NULL; const char **params = NULL; if (nParams) { if (m_name.empty()) paramTypes = &m_paramTypes[0]; params = &m_params[0]; paramLengths = &m_paramLengths[0]; paramFormats = &m_paramFormats[0]; } const char *api = NULL; #ifndef WINDOWS SchedulerSwitcher switcher(m_scheduler); #endif if (m_name.empty()) { #ifndef WINDOWS if (m_scheduler) { api = "PQsendQueryParams"; if (!PQsendQueryParams(conn, m_command.c_str(), nParams, paramTypes, params, paramLengths, paramFormats, m_resultFormat)) throwException(conn); flush(conn, m_scheduler); next.reset(nextResult(conn, m_scheduler), &PQclear); while (next) { result = next; next.reset(nextResult(conn, m_scheduler), &PQclear); if (next) { ExecStatusType status = PQresultStatus(next.get()); MORDOR_LOG_VERBOSE(g_log) << conn << "PQresultStatus(" << next.get() << "): " << PQresStatus(status); switch (status) { case PGRES_COMMAND_OK: case PGRES_TUPLES_OK: break; default: throwException(next.get()); MORDOR_NOTREACHED(); } } } } else #endif { api = "PQexecParams"; result.reset(PQexecParams(conn, m_command.c_str(), nParams, paramTypes, params, paramLengths, paramFormats, m_resultFormat), &PQclear); } } else { #ifndef WINDOWS if (m_scheduler) { api = "PQsendQueryPrepared"; if (!PQsendQueryPrepared(conn, m_name.c_str(), nParams, params, paramLengths, paramFormats, 1)) throwException(conn); flush(conn, m_scheduler); next.reset(nextResult(conn, m_scheduler), &PQclear); while (next) { result = next; next.reset(nextResult(conn, m_scheduler), &PQclear); if (next) { ExecStatusType status = PQresultStatus(next.get()); MORDOR_LOG_VERBOSE(g_log) << conn << "PQresultStatus(" << next.get() << "): " << PQresStatus(status); switch (status) { case PGRES_COMMAND_OK: case PGRES_TUPLES_OK: break; default: throwException(next.get()); MORDOR_NOTREACHED(); } } } } else #endif { api = "PQexecPrepared"; result.reset(PQexecPrepared(conn, m_name.c_str(), nParams, params, paramLengths, paramFormats, m_resultFormat), &PQclear); } } if (!result) throwException(conn); ExecStatusType status = PQresultStatus(result.get()); MORDOR_ASSERT(api); MORDOR_LOG_VERBOSE(g_log) << conn << " " << api << "(\"" << m_command << m_name << "\", " << nParams << "), PQresultStatus(" << result.get() << "): " << PQresStatus(status); switch (status) { case PGRES_COMMAND_OK: case PGRES_TUPLES_OK: return Result(result); default: throwException(result.get()); MORDOR_NOTREACHED(); } }
void run_indexing(int rank, int interpolation, PGconn *conn, int num_threads, struct index_thread_data * thread_data, const char *structuredoutputfile) { int tuples, count, sleepcount; pthread_mutex_t count_mutex = PTHREAD_MUTEX_INITIALIZER; time_t rankStartTime; int rankTotalTuples; int rankCountTuples; float rankPerSecond; PGresult * resSectors; PGresult * resPlaces; PGresult * resNULL; int i; int iSector; int iResult; const char *paramValues[2]; int paramLengths[2]; int paramFormats[2]; uint32_t paramRank; uint32_t paramSector; uint32_t sector; xmlTextWriterPtr writer; pthread_mutex_t writer_mutex = PTHREAD_MUTEX_INITIALIZER; // Create the output file writer = NULL; if (structuredoutputfile) { writer = nominatim_exportXMLStart(structuredoutputfile); } if (interpolation) { fprintf(stderr, "Starting interpolation lines (location_property_osmline)\n"); } else { fprintf(stderr, "Starting rank %d\n", rank); } rankCountTuples = 0; rankPerSecond = 0; paramRank = PGint32(rank); paramValues[0] = (char *)¶mRank; paramLengths[0] = sizeof(paramRank); paramFormats[0] = 1; if (interpolation) { resSectors = PQexecPrepared(conn, "index_sectors_osmline", 0, NULL, 0, NULL, 1); } else { resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1); } if (PQresultStatus(resSectors) != PGRES_TUPLES_OK) { fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn)); PQclear(resSectors); exit(EXIT_FAILURE); } if (PQftype(resSectors, 0) != PG_OID_INT4) { fprintf(stderr, "Sector value has unexpected type\n"); PQclear(resSectors); exit(EXIT_FAILURE); } if (PQftype(resSectors, 1) != PG_OID_INT8) { fprintf(stderr, "Sector value has unexpected type\n"); PQclear(resSectors); exit(EXIT_FAILURE); } rankTotalTuples = 0; for (iSector = 0; iSector < PQntuples(resSectors); iSector++) { rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))); } rankStartTime = time(0); for (iSector = 0; iSector <= PQntuples(resSectors); iSector++) { if (iSector > 0) { resPlaces = PQgetResult(conn); if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK) { fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn)); PQclear(resPlaces); exit(EXIT_FAILURE); } if (PQftype(resPlaces, 0) != PG_OID_INT8) { fprintf(stderr, "Place_id value has unexpected type\n"); PQclear(resPlaces); exit(EXIT_FAILURE); } resNULL = PQgetResult(conn); if (resNULL != NULL) { fprintf(stderr, "Unexpected non-null response\n"); exit(EXIT_FAILURE); } } if (iSector < PQntuples(resSectors)) { sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0))); // fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)))); // Get all the place_id's for this sector paramRank = PGint32(rank); paramSector = PGint32(sector); if (rankTotalTuples-rankCountTuples < num_threads*1000) { // no sectors if (interpolation) { iResult = PQsendQueryPrepared(conn, "index_nosector_places_osmline", 0, NULL, 0, NULL, 1); } else { paramValues[0] = (char *)¶mRank; paramLengths[0] = sizeof(paramRank); paramFormats[0] = 1; iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1); } } else { if (interpolation) { iResult = PQsendQueryPrepared(conn, "index_sector_places_osmline", 1, paramValues, paramLengths, paramFormats, 1); paramValues[0] = (char *)¶mSector; paramLengths[0] = sizeof(paramSector); paramFormats[0] = 1; } else { paramValues[0] = (char *)¶mRank; paramLengths[0] = sizeof(paramRank); paramFormats[0] = 1; paramValues[1] = (char *)¶mSector; paramLengths[1] = sizeof(paramSector); paramFormats[1] = 1; iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1); } } if (!iResult) { fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn)); PQclear(resPlaces); exit(EXIT_FAILURE); } } if (iSector > 0) { count = 0; rankPerSecond = 0; tuples = PQntuples(resPlaces); if (tuples > 0) { // Spawn threads for (i = 0; i < num_threads; i++) { thread_data[i].res = resPlaces; thread_data[i].tuples = tuples; thread_data[i].count = &count; thread_data[i].count_mutex = &count_mutex; thread_data[i].writer = writer; thread_data[i].writer_mutex = &writer_mutex; if (interpolation) { thread_data[i].table = 0; // use interpolations table } else { thread_data[i].table = 1; // use placex table } pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]); } // Monitor threads to give user feedback sleepcount = 0; while (count < tuples) { usleep(1000); // Aim for one update per second if (sleepcount++ > 1000) { rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1); if(interpolation) { fprintf(stderr, " Done %i in %i @ %f per second - Interpolation lines ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond); } else { fprintf(stderr, " Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond); } sleepcount = 0; } } // Wait for everything to finish for (i = 0; i < num_threads; i++) { pthread_join(thread_data[i].thread, NULL); } rankCountTuples += tuples; } // Finished sector rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1); fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond); PQclear(resPlaces); } if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors)) { iSector = PQntuples(resSectors) - 1; } } // Finished rank fprintf(stderr, "\r Done %i in %i @ %f per second - FINISHED\n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond); PQclear(resSectors); }