/// Record an aggregation function. Return a math term of the type
/// variable to the caller so the caller can continue to build up a larger
/// expression. For simplicity, the variable name is simply "__hhh", where
/// "hhh" is the size of aggr_ in hexadecimal.
///
/// @note This function takes charge of expr. It will free the object if
/// the object is not passed on to other operations. This can happen when
/// the particular variable appeared already in the select clause.
ibis::math::variable*
ibis::selectClause::addAgregado(ibis::selectClause::AGREGADO agr,
ibis::math::term *expr) {
if (agr != ibis::selectClause::NIL_AGGR &&
hasAggregation(expr)) {
LOGGER(ibis::gVerbose >= 0)
<< "Warning -- selectClause can not have aggregations inside "
"another aggregation operation (" << *expr << ')';
throw "selectClause::addAgregado failed due to nested aggregations"
IBIS_FILE_LINE;
}
const unsigned end = atms_.size();
LOGGER(ibis::gVerbose > 5)
<< "selectClause::addAgregado -- adding term " << end << ": "
<< aggDescription(agr, expr);
if (expr->termType() != ibis::math::VARIABLE) {
aggr_.push_back(agr);
atms_.push_back(expr);
std::ostringstream oss;
oss << "__" << std::hex << end;
ordered_[oss.str()] = end;
return new ibis::selectClause::variable(oss.str().c_str(), this);
}
else {
ibis::math::variable *var = static_cast<ibis::math::variable*>(expr);
ibis::selectClause::StringToInt::const_iterator it =
ordered_.find(var->variableName());
if (it == ordered_.end()) { // no in the existing list
aggr_.push_back(agr);
atms_.push_back(expr);
ordered_[var->variableName()] = end;
if (agr != ibis::selectClause::NIL_AGGR) {
std::ostringstream oss;
oss << "__" << std::hex << end;
ordered_[oss.str()] = end;
return new ibis::selectClause::variable
(oss.str().c_str(), this);
}
else {
return var->dup();
}
}
else if (agr != aggr_[it->second]) { // new aggregation
aggr_.push_back(agr);
atms_.push_back(expr);
if (agr != ibis::selectClause::NIL_AGGR) {
std::ostringstream oss;
oss << "__" << std::hex << end;
ordered_[oss.str()] = end;
return new ibis::selectClause::variable
(oss.str().c_str(), this);
}
else {
ordered_[var->variableName()] = end;
return var->dup();
}
}
else { // the variable has appeared before
delete expr;
std::ostringstream oss;
oss << "__" << std::hex << it->second;
return new ibis::selectClause::variable(oss.str().c_str(), this);
}
}
return 0;
} // ibis::selectClause::addAgregado
/**
* Processes a File Receive message from the client,
* performs and finalizes the operation
*
* @param proc_data_arg data structure with connection parameters
* and received message
*/
void process_file_receive( PROCESS_DATA_T * proc_data ) {
const int NOT_FOUND = -1;
char * filename = NULL;
char l_msg[4096];
char * request = NULL;
char * response = NULL;
int search_filename_res = 0;
unsigned long response_len = 0;
int result = RESULT_UNDEFINED;
const int param_len = (proc_data->received_msg_len - HEADER_LEN - strlen(PARAM_FILENAME));
// Copies the request to a safe buffer for searching
request = (char*)malloc(sizeof(char) * proc_data->received_msg_len + 1);
memcpy(request, proc_data->received_message, proc_data->received_msg_len);
request[proc_data->received_msg_len] = '\0';
search_filename_res = STR_SEARCH( request, PARAM_FILENAME, HEADER_LEN);
if ( search_filename_res == NOT_FOUND ) {
result = RESULT_INVALID_REQUEST;
goto END_PROCESS_FILE_RECEIVE;
}
// Gets name of file to send
filename = (char*)malloc(sizeof(char) * proc_data->received_msg_len);
memcpy( filename, &(request[ ( search_filename_res + strlen(PARAM_FILENAME) ) ]), param_len );
filename[param_len] = '\0';
sprintf(l_msg, "A request has been received to send the following file: %s", filename);
LOGGER(__FUNCTION__, l_msg);
//
// Find, pack, and encode file
//
{
long long filesize = file_size(filename);
if ( ! file_exists(filename) || filesize <= 0 ) {
LOGGER(__FUNCTION__, "No files have been found for the specified mask.");
result = RESULT_FILE_NOT_FOUND;
} else {
char gzip_output[256];
char b64_output[256];
sprintf(gzip_output, ".\\%d-%lu.gz", proc_data->process_id, GetTickCount() );
sprintf(b64_output, ".\\%d-%lu.b64", proc_data->process_id, GetTickCount() );
if ( gz_pack_file(filename, gzip_output) == TRUE )
{
if ( base64_process_file('e', gzip_output, b64_output, filesize) == TRUE )
{
unsigned long bytes_read;
char * buffer = NULL;
FILE * f = NULL;
filesize = file_size(b64_output);
buffer = (char*)malloc(sizeof(char) * filesize + 1);
memset(buffer, 0x00, filesize + 1);
f = fopen(b64_output, "r");
if (f != NULL)
{
bytes_read = fread(buffer, 1, filesize, f);
sprintf(l_msg, "%lu bytes read from file %s to process and send.", bytes_read, b64_output);
LOGGER(__FUNCTION__, l_msg);
fclose(f);
result = RESULT_SUCCESS;
// Generates response message with file content
response = message_file_receive_response( result, strlen(buffer), buffer, &response_len );
// Sends a File Receive response message
if ( !process_outgoing_message( proc_data->connection, response, response_len ) ) {
LOGGER(__FUNCTION__, "File Receive message could not be sent.");
}
// Deletes generated temporary files
remove(gzip_output);
remove(b64_output);
// Frees response
free(response);
} else {
sprintf(l_msg, "Error opening file (%s)", b64_output);
LOGGER(__FUNCTION__, l_msg);
result = RESULT_FILE_READ_ERROR;
}
// Frees buffer
free(buffer);
} else {
sprintf(l_msg, "Error encoding file (%s)", gzip_output);
LOGGER(__FUNCTION__, l_msg);
result = RESULT_FILE_ENCODE_ERROR;
}
} else {
sprintf(l_msg, "Error packing file (%s)", filename);
LOGGER(__FUNCTION__, l_msg);
result = RESULT_FILE_COMPRESS_ERROR;
}
}
}
END_PROCESS_FILE_RECEIVE:
// Cleanup
if (request != NULL) {
free(request);
}
if (filename != NULL) {
free(filename);
}
if (result == RESULT_SUCCESS) {
return;
}
else {
// Generates response
response = message_file_receive_response( result, 0, NULL, &response_len );
// Sends a File Receive response message
if ( !process_outgoing_message( proc_data->connection, response, response_len ) ) {
LOGGER(__FUNCTION__, "File Receive response message could not be sent.");
}
// Libera la respuesta
free(response);
}
return;
}
/**
* Processes a File Delete message from the client,
* performs and finalizes the operation
*
* @param proc_data_arg data structure with connection parameters
* and received message
*/
void process_file_delete( PROCESS_DATA_T * proc_data ) {
const int NOT_FOUND = -1;
char * filename = NULL;
char l_msg[4096];
char * request = NULL;
char * response = NULL;
unsigned long response_len = 0;
int search_filename_res = 0;
int result = RESULT_UNDEFINED;
const int param_len = (proc_data->received_msg_len - HEADER_LEN - strlen(PARAM_FILENAME));
// Copies the request to a safe buffer for searching
request = (char*)malloc(sizeof(char) * proc_data->received_msg_len + 1);
memcpy(request, proc_data->received_message, proc_data->received_msg_len);
request[proc_data->received_msg_len] = '\0';
// Gets name of file to delete
search_filename_res = STR_SEARCH( proc_data->received_message, PARAM_FILENAME, HEADER_LEN);
if ( search_filename_res == NOT_FOUND ) {
result = RESULT_INVALID_REQUEST;
goto END_PROCESS_FILE_DELETE;
}
filename = (char*)malloc(sizeof(char) * proc_data->received_msg_len);
memcpy( filename, &(request[ ( search_filename_res + strlen(PARAM_FILENAME) ) ]), param_len );
filename[param_len] = '\0';
sprintf(l_msg, "A request has been received to delete the file: %s", filename);
LOGGER(__FUNCTION__, l_msg);
if (file_exists(filename) == TRUE) {
if (file_delete(filename) == TRUE) {
result = RESULT_SUCCESS;
}
else {
result = RESULT_FILE_DELETE_ERROR;
}
}
else {
result = RESULT_FILE_NOT_FOUND;
}
// Generates response message
response = message_file_delete_response( result, &response_len );
// Sends response message
if ( !process_outgoing_message( proc_data->connection, response, response_len ) ) {
LOGGER(__FUNCTION__, "File Receive response message could not be sent.");
}
END_PROCESS_FILE_DELETE:
// Cleanup
if (request != NULL) {
free(request);
}
if (response != NULL) {
free(response);
}
if (filename != NULL) {
free(filename);
}
return;
}
void stateOperatorControl() {
// DRIVING
move = stick->GetRawAxis(1) * -1.0;
rotate = stick->GetRawAxis(4) * -1.0;
// Deadband
if (fabs(move) < 0.1) {
move = 0.0;
}
if (fabs(rotate) < 0.15) {
rotate = 0.0;
}
drive->ArcadeDrive(move, rotate, false);
// Joystick Buttons
bool button4 = stick->GetRawButton(4);
bool button1 = stick->GetRawButton(1);
bool button5 = stick->GetRawButton(5);
bool button6 = stick->GetRawButton(6);
bool button3 = stick->GetRawButton(3);
// Manual Gatherer
if (stick->GetRawAxis(2) != 0) {
gatherSpeed = stick->GetRawAxis(2);
LOGGER(DEBUG) << "[stateOperatorControl] Gather Angle:" << gatherPIDSource.PIDGet();
}
else if (stick->GetRawAxis(3) != 0) {
gatherSpeed = stick->GetRawAxis(3) * -1;
LOGGER(DEBUG) << "[stateOperatorControl] Gather Angle:" << gatherPIDSource.PIDGet();
}
else {
gatherSpeed = 0.0;
}
gatherer->Set(gatherSpeed);
// Launch Angle
double launcherAngle = launchPIDSource.PIDGet();
if (button5 && !button6 && (launcherAngle < kLaunchMaxAngle)) {
elevator->Set(-0.5); // Up
LOGGER(DEBUG) << "[stateOperatorControl] Launcher Angle:" << launcherAngle;
} else if (button6 && !button5 && (launcherAngle > kLaunchMinAngle)) {
LOGGER(DEBUG) << "[stateOperatorControl] Launcher Angle:" << launcherAngle;
elevator->Set(0.5); // Down
} else {
elevator->Set(0.0);
}
// Auto-Gather
if (button3 && !lastButton3) {
wheelsGathererIn = !wheelsGathererIn;
gatherController->SetSetpoint(kGatherAngle);
gatherController->Enable();
}
if (wheelsGathererIn) {
gathererWheels->Set(1.0);
gatherer->Set(gatherPIDOutput.correction);
LOGGER(DEBUG) << "[stateOperatorControl] Gather Correction:" << gatherPIDOutput.correction
<< " Gather Angle: " << gatherPIDSource.PIDGet();
} else {
gathererWheels->Set(0.0);
gatherController->Disable();
}
if (button4 && !lastButton4) {
stateTimer = 0;
robotState = kCentering;
shootingHigh = true;
}
if (button1 && !lastButton1) {
stateTimer = 0;
robotState = kLaunching;
shootingHigh = true;
}
lastButton4 = button4;
lastButton1 = button1;
lastButton3 = button3;
}
MCFG_CPU_PERIODIC_INT_DRIVER(midcoin24cdjuke_state, irq0_line_hold, 500)
MCFG_DEFAULT_LAYOUT(layout_24cdjuke)
MCFG_DEVICE_ADD("ic11", I8255A, 0)
MCFG_I8255_IN_PORTA_CB(IOPORT("MD1"))
MCFG_I8255_IN_PORTB_CB(IOPORT("MD2"))
MCFG_I8255_IN_PORTC_CB(IOPORT("MD3"))
MCFG_DEVICE_ADD("ic25", I8255A, 0)
MCFG_I8255_IN_PORTB_CB(IOPORT("PB"))
MCFG_I8255_IN_PORTC_CB(READ8(midcoin24cdjuke_state, kb_row_r))
MCFG_I8255_OUT_PORTC_CB(WRITE8(midcoin24cdjuke_state, kb_col_w))
MCFG_DEVICE_ADD("ic31", I8255A, 0)
MCFG_I8255_OUT_PORTB_CB(LOGGER("PPI8255 - unmapped write port B", 0))
MCFG_I8255_IN_PORTC_CB(IOPORT("MD4"))
MACHINE_CONFIG_END
ROM_START( 24cdjuke )
ROM_REGION( 0x4000, "maincpu", 0 )
ROM_LOAD( "1.ic5", 0x0000, 0x4000, CRC(df2419ad) SHA1(dd9dd85011d46581dccabcfdb5959a8b018df937) )
ROM_REGION16_LE( 0x200, "charset", 0 )
ROM_LOAD16_BYTE( "dm74ls471n.ic20", 0x000, 0x100, CRC(d05765e6) SHA1(119ec6ca1a4afa0ea6ab1020ba2a8b02fd434e3f) )
ROM_LOAD16_BYTE( "dm74ls471n.ic21", 0x001, 0x100, CRC(e12d5a04) SHA1(be52ee4e4a5ea225fce39c759645a7cf49cea370) )
// MAB8441T-T042 internal ROM?
ROM_REGION( 0x80000, "misc", 0 )
ROM_LOAD( "m1-7611a-5.ic27", 0x000, 0x100, CRC(29b068e8) SHA1(477e2445c58b7d14c56a3ad4050eb22474d56005) )
/**
* Checks if a socket is available for reading and/or writing
*
* @param timeout timeout, or 0 for blocking
* @param socket socket to check state on
* @param operation_type operation type: S_READ, S_WRITE, S_RW
*
* @return type of operation the socket is available for,
* can be S_READ, S_WRITE, S_RW,
* or -1 if an error occurred
*/
int socket_select(int timeout, SOCKET_T * select_socket, int operation_type) {
fd_set* readfds = NULL;
fd_set* writefds = NULL;
struct timeval tval_timeout;
char buffer[1024];
int retval = 0;
int res;
// Validates parameters
if ( ( select_socket == NULL ) || ( (operation_type != S_READ) && (operation_type != S_WRITE) && (operation_type != S_RW) ) ) {
sprintf(buffer, "socket_select fail: invalid parameters");
LOGGER(__FUNCTION__, buffer);
return -1;
}
// Configures timeout
tval_timeout.tv_sec = timeout;
tval_timeout.tv_usec = 0;
// Allocates memory for sets
readfds = (fd_set*)malloc(sizeof(fd_set));
writefds = (fd_set*)malloc(sizeof(fd_set));
// Initializes sets
FD_ZERO(readfds);
FD_ZERO(writefds);
// Locks the socket's mutex
MUTEX_LOCK(select_socket->mutex);
// If has to check for reading
if (operation_type & S_READ) {
// Adds socket to the read set
FD_SET( select_socket->handle, readfds );
}
// If has to check for writing
if (operation_type & S_WRITE) {
// Adds socket to the write set
FD_SET(select_socket->handle, writefds);
}
// Calls select function
res = select( (select_socket->handle)+1, readfds, writefds, NULL, &tval_timeout );
if (res == -1) {
sprintf(buffer, "select failed with error: %d\n", errno);
LOGGER(__FUNCTION__, buffer);
retval = -1;
goto end_select;
}
// If socket is on the read result set
// adds to return value
if ( FD_ISSET(select_socket->handle, readfds ) ) {
retval += S_READ;
}
// If socket is on the write result set
// adds to return value
if ( FD_ISSET(select_socket->handle, writefds ) ) {
retval += S_WRITE;
}
end_select:
// Unlocks the socket's mutex
MUTEX_UNLOCK(select_socket->mutex);
// Frees allocated memory
free(readfds);
free(writefds);
return retval;
}
/**
* Creates a new socket and connects to server if it is a client,
* or does the binding and starts listening on a port if it is server.
* Returns a reference to the newly created connection.
*
* @param side 0 if it is a client, 1 if it is a server
* @param addr address to connect for clients or to listen on
* for server, if NULL for server listens on INADDR_ANY
* @param port port to connect or to listen on
* @param max_connections max number of connections that the server can
* accept. does not apply for clients.
* @param nonblocking TRUE if nonblocking is desired, otherwise FALSE
*
* @return pointer to the newly created socket, on error
* returns NULL
*/
SOCKET_T* socket_create(int side, char* addr, int port, int max_connections, int nonblocking) {
SOCKET_T* new_socket = NULL;
struct sockaddr_in service;
char buffer[_BUFFER_SIZE_S];
int res;
int socket_handle;
// Validates paramter
if (side < 0 || side > 1) {
sprintf(buffer, "socket_create fail: parametros no validos");
return NULL;
}
// Configurates the service
service.sin_family = AF_INET;
service.sin_port = htons((unsigned short)port);
if (addr == NULL) {
service.sin_addr.s_addr = htonl(INADDR_ANY);
} else {
service.sin_addr.s_addr = inet_addr(addr);
}
//
// Creates socket and does the binding and listening
// operations for server or connect for clients
//
// Creates socket
if (nonblocking == TRUE) {
socket_handle = socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0);
} else {
socket_handle = socket(AF_INET, SOCK_STREAM, 0);
}
if (socket_handle == -1) {
sprintf(buffer, "socket failed with error: %d\n", errno);
LOGGER(__FUNCTION__, buffer);
return NULL;
}
// If it is server
if (side == 1) {
// Binds the socket to the port
res = bind (socket_handle, (struct sockaddr *) &service, sizeof (service));
if (res == -1)
{
sprintf(buffer, "socket failed with error: %d\n", errno);
LOGGER(__FUNCTION__, buffer);
shutdown(socket_handle, SHUT_RDWR);
return NULL;
}
// Starts listening on the port
res = listen(socket_handle, max_connections);
if (res == -1) {
sprintf(buffer, "listen fallo con el error: %d\n", errno);
LOGGER(__FUNCTION__, buffer);
shutdown(socket_handle, SHUT_RDWR);
return NULL;
}
}
// If it is client
else {
// Connects to the server
res = connect(socket_handle, (struct sockaddr *) &service, sizeof (service));
if ( res == -1) {
// Unless connection is in progress
if ( errno != EINPROGRESS ) {
sprintf(buffer, "connect failed with error: %d\n", errno);
LOGGER(__FUNCTION__, buffer);
shutdown(socket_handle, SHUT_RDWR);
return NULL;
}
}
}
// Configures timeouts for read/write
{
struct timeval timeout;
timeout.tv_sec = RW_TIMEOUT;
timeout.tv_usec = 0;
res = setsockopt(socket_handle, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(timeout));
if (res == -1) {
sprintf(buffer, "setsockopt failed with error: %d\n", errno);
LOGGER(__FUNCTION__, buffer);
}
res = setsockopt(socket_handle, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(timeout));
if (res == -1) {
sprintf(buffer, "setsockopt failed with error: %d\n", errno);
LOGGER(__FUNCTION__, buffer);
}
}
// Allocates space for the socket structure
new_socket = malloc(sizeof( SOCKET_T ));
memset(new_socket, 0x00, sizeof( SOCKET_T ));
// Creates mutex for thread-safe socket
new_socket->mutex = (MUTEX_T*)malloc(sizeof(struct _mutex_t) );
memset(new_socket->mutex, 0x00, sizeof(struct _mutex_t));
MUTEX_CREATE(&new_socket->mutex);
// Copies handle to the socket structure
new_socket->handle = socket_handle;
return new_socket;
}
ibis::table* ibis::jRange::select(const char *sstr) const {
if (nrows < 0) {
int64_t ierr = count();
if (ierr < 0) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- jRange::count failed with error code"
<< ierr;
return 0;
}
}
if (sstr == 0 || *sstr == 0) { // default
std::string tn = ibis::util::shortName(desc_.c_str());
return new ibis::tabula(tn.c_str(), desc_.c_str(), nrows);
}
ibis::selectClause sel(sstr);
uint32_t features=0; // 1: arithmetic operation, 2: aggregation
// use a barrel to collect all unique names
ibis::math::barrel brl;
for (uint32_t j = 0; j < sel.aggSize(); ++ j) {
const ibis::math::term* t = sel.aggExpr(j);
brl.recordVariable(t);
if (t->termType() != ibis::math::VARIABLE &&
t->termType() != ibis::math::NUMBER &&
t->termType() != ibis::math::STRING) {
features |= 1; // arithmetic operation
}
if (sel.getAggregator(j) != ibis::selectClause::NIL_AGGR) {
features |= 2; // aggregation
}
}
// convert the barrel into a stringArray for processing
ibis::table::stringArray sl;
sl.reserve(brl.size());
for (unsigned j = 0; j < brl.size(); ++ j) {
const char* str = brl.name(j);
if (*str != 0) {
if (str[0] != '_' || str[1] != '_')
sl.push_back(str);
}
}
std::unique_ptr<ibis::table> res1(select(sl));
if (res1.get() == 0 || res1->nRows() == 0 || res1->nColumns() == 0 ||
features == 0)
return res1.release();
if (ibis::gVerbose > 2) {
ibis::util::logger lg;
lg() << "jRange::select(" << sstr << ", " << desc_
<< ") produced the first intermediate table:\n";
res1->describe(lg());
}
if ((features & 1) != 0) { // arithmetic computations
res1.reset(static_cast<const ibis::bord*>(res1.get())->evaluateTerms
(sel, desc_.c_str()));
if (res1.get() != 0) {
if (ibis::gVerbose > 2) {
ibis::util::logger lg;
lg() << "jRange::select(" << sel << ", " << desc_
<< ") produced the second intermediate table:\n";
res1->describe(lg());
}
}
else {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- jRange::select(" << sel
<< ") failed to evaluate the arithmetic expressions";
return 0;
}
}
if ((features & 2) != 0) { // aggregation operations
res1.reset(static_cast<const ibis::bord*>(res1.get())->groupby(sel));
if (res1.get() != 0) {
if (ibis::gVerbose > 2) {
ibis::util::logger lg;
lg() << "jRange::select(" << *sel_ << ", " << desc_
<< ") produced the third intermediate table:\n";
res1->describe(lg());
}
}
else {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- jRange::select(" << *sel_
<< ") failed to evaluate the aggregations";
}
}
return res1.release();
} // ibis::jRange::select
ibis::table*
ibis::jRange::select(const ibis::table::stringArray& colnames) const {
ibis::table *res = 0;
if (nrows < 0) {
int64_t ierr = count();
if (ierr < 0) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- jRange::count failed with error code"
<< ierr;
return res;
}
}
if (valr_ == 0 || orderr_ == 0 || vals_ == 0 || orders_ == 0 ||
orderr_->size() != maskr_.cnt() || orders_->size() != masks_.cnt()) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- jRange::select failed to evaluate the join";
return res;
}
if (colnames.empty() || nrows == 0) {
std::string nm = ibis::util::shortName(desc_);
res = new ibis::tabula(nm.c_str(), desc_.c_str(), nrows);
return res;
}
const uint32_t ncols = colnames.size();
std::string evt;
evt = "select ";
evt += colnames[0];
for (uint32_t j = 1; j < ncols; ++ j) {
evt += ", ";
evt += colnames[j];
}
if ((desc_[0] != 'F' && desc_[0] != 'f') ||
(desc_[1] != 'R' && desc_[1] != 'r') ||
(desc_[2] != 'O' && desc_[2] != 'o') ||
(desc_[3] != 'M' && desc_[3] != 'm'))
evt += " for ";
else
evt += ' ';
evt += desc_;
ibis::util::timer mytimer(evt.c_str());
std::map<const char*, uint32_t, ibis::lessi> namesToPos;
std::vector<uint32_t> ipToPos(colnames.size());
std::vector<const ibis::column*> ircol, iscol;
std::vector<const ibis::dictionary*> cats(colnames.size(), 0);
// identify the names from the two data partitions
for (uint32_t j = 0; j < ncols; ++ j) {
ipToPos[j] = ncols+1;
const char* cn = colnames[j];
std::string tname;
while (*cn != 0 && *cn != '.') {
tname += *cn;
++ cn;
}
if (*cn == '.') {
++ cn;
}
else { // did not find '.'
tname.erase();
cn = colnames[j];
}
int match = -1; // 0 ==> partr_, 1 ==> parts_
if (! tname.empty()) {
match = frm_->position(tname.c_str());
if (match >= static_cast<long>(frm_->size())) {
if (stricmp(tname.c_str(), partr_.name()) == 0) {
match = 0;
}
else if (stricmp(tname.c_str(), parts_.name()) == 0) {
match = 1;
}
}
}
if (match == 0) {
const ibis::column *col = partr_.getColumn(cn);
if (col != 0) {
namesToPos[colnames[j]] = j;
ipToPos[j] = ircol.size();
ircol.push_back(col);
if (col->type() == ibis::CATEGORY) {
const ibis::category *cat =
static_cast<const ibis::category*>(col);
cats[j] = cat->getDictionary();
}
else if (col->type() == ibis::UINT) {
const ibis::bord::column *bc =
dynamic_cast<const ibis::bord::column*>(col);
if (bc != 0) {
cats[j] = bc->getDictionary();
}
}
}
else {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- " << evt << " can not find column named \""
<< colnames[j] << "\" in data partition \"" << partr_.name()
<< "\"";
return res;
}
}
else if (match == 1) {
const ibis::column *col = parts_.getColumn(cn);
if (col != 0) {
namesToPos[colnames[j]] = j;
ipToPos[j] = ncols - iscol.size();
iscol.push_back(col);
if (col->type() == ibis::CATEGORY) {
const ibis::category *cat =
static_cast<const ibis::category*>(col);
cats[j] = cat->getDictionary();
}
else if (col->type() == ibis::UINT) {
const ibis::bord::column *bc =
dynamic_cast<const ibis::bord::column*>(col);
if (bc != 0) {
cats[j] = bc->getDictionary();
}
}
}
else {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- " << evt << " can not find column named \""
<< colnames[j] << "\" in data partition \""
<< parts_.name() << "\"";
return res;
}
}
else { // not prefixed with a data partition name
cn = colnames[j];
const ibis::column* col = partr_.getColumn(cn);
if (col != 0) {
ipToPos[j] = ircol.size();
ircol.push_back(col);
if (col->type() == ibis::CATEGORY) {
const ibis::category *cat =
static_cast<const ibis::category*>(col);
cats[j] = cat->getDictionary();
}
else if (col->type() == ibis::UINT) {
const ibis::bord::column *bc =
dynamic_cast<const ibis::bord::column*>(col);
if (bc != 0) {
cats[j] = bc->getDictionary();
}
}
LOGGER(ibis::gVerbose > 3)
<< evt << " encountered a column name ("
<< colnames[j] << ") that does not start with a data "
"partition name, assume it is for \"" << partr_.name()
<< "\"";
}
else {
col = parts_.getColumn(cn);
if (col != 0) {
ipToPos[j] = ncols - iscol.size();
iscol.push_back(col);
if (col->type() == ibis::CATEGORY) {
const ibis::category *cat =
static_cast<const ibis::category*>(col);
cats[j] = cat->getDictionary();
}
else if (col->type() == ibis::UINT) {
const ibis::bord::column *bc =
dynamic_cast<const ibis::bord::column*>(col);
if (bc != 0) {
cats[j] = bc->getDictionary();
}
}
LOGGER(ibis::gVerbose > 1)
<< evt << " encountered a column name (" << colnames[j]
<< ") that does not start with a data partition name, "
"assume it is for \"" << parts_.name() << "\"";
}
else {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- " << evt << " encountered a name ("
<< colnames[j] << ") that does not start with a data "
"partition name";
return res;
}
}
}
} // for (uint32_t j = 0; j < ncols;
LOGGER(ibis::gVerbose > 3)
<< evt << " -- found " << ircol.size()
<< " column" << (ircol.size() > 1 ? "s" : "") << " from "
<< partr_.name() << " and " << iscol.size() << " column"
<< (iscol.size() > 1 ? "s" : "") << " from " << parts_.name();
// change Pos values for columns in S to have offset ircol.size()
for (uint32_t j = 0; j < ncols; ++j) {
if (ipToPos[j] <= ncols && ipToPos[j] >= ircol.size())
ipToPos[j] = (ncols - ipToPos[j]) + ircol.size();
}
ibis::table::typeArray rtypes(ircol.size(), ibis::UNKNOWN_TYPE);
ibis::table::bufferArray rbuff(ircol.size(), 0);
IBIS_BLOCK_GUARD(ibis::table::freeBuffers, ibis::util::ref(rbuff),
ibis::util::ref(rtypes));
ibis::table::typeArray stypes(iscol.size(), ibis::UNKNOWN_TYPE);
ibis::table::bufferArray sbuff(iscol.size(), 0);
IBIS_BLOCK_GUARD(ibis::table::freeBuffers, ibis::util::ref(sbuff),
ibis::util::ref(stypes));
bool sane = true;
// retrieve values from r_
for (uint32_t j = 0; sane && j < ircol.size(); ++ j) {
rtypes[j] = ircol[j]->type();
switch (ircol[j]->type()) {
case ibis::BYTE:
rbuff[j] = ircol[j]->selectBytes(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<signed char>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::UBYTE:
rbuff[j] = ircol[j]->selectUBytes(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<unsigned char>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::SHORT:
rbuff[j] = ircol[j]->selectShorts(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<int16_t>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::USHORT:
rbuff[j] = ircol[j]->selectUShorts(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<uint16_t>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::INT:
rbuff[j] = ircol[j]->selectInts(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<int32_t>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::UINT:
rbuff[j] = ircol[j]->selectUInts(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<uint32_t>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::LONG:
rbuff[j] = ircol[j]->selectLongs(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<int64_t>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::ULONG:
rbuff[j] = ircol[j]->selectULongs(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<uint64_t>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::FLOAT:
rbuff[j] = ircol[j]->selectFloats(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<float>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::DOUBLE:
rbuff[j] = ircol[j]->selectDoubles(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<double>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::TEXT:
case ibis::CATEGORY:
rbuff[j] = ircol[j]->selectStrings(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<std::vector<std::string>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
default:
sane = false;
rbuff[j] = 0;
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::select does not support column "
"type " << ibis::TYPESTRING[(int)ircol[j]->type()]
<< " (name = " << partr_.name() << "." << ircol[j]->name()
<< ")";
break;
}
}
if (! sane) {
return res;
}
// retrieve values from parts_
for (uint32_t j = 0; sane && j < iscol.size(); ++ j) {
stypes[j] = iscol[j]->type();
switch (iscol[j]->type()) {
case ibis::BYTE:
sbuff[j] = iscol[j]->selectBytes(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<signed char>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::UBYTE:
sbuff[j] = iscol[j]->selectUBytes(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<unsigned char>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::SHORT:
sbuff[j] = iscol[j]->selectShorts(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<int16_t>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::USHORT:
sbuff[j] = iscol[j]->selectUShorts(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<uint16_t>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::INT:
sbuff[j] = iscol[j]->selectInts(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<int32_t>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::UINT:
sbuff[j] = iscol[j]->selectUInts(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<uint32_t>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::LONG:
sbuff[j] = iscol[j]->selectLongs(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<int64_t>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::ULONG:
sbuff[j] = iscol[j]->selectULongs(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<uint64_t>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::FLOAT:
sbuff[j] = iscol[j]->selectFloats(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<float>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::DOUBLE:
sbuff[j] = iscol[j]->selectDoubles(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<double>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::TEXT:
case ibis::CATEGORY:
sbuff[j] = iscol[j]->selectStrings(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<std::vector<std::string>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
default:
sane = false;
sbuff[j] = 0;
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::select does not support column "
"type " << ibis::TYPESTRING[(int)iscol[j]->type()]
<< " (name = " << parts_.name() << "." << iscol[j]->name()
<< ")";
break;
}
}
if (! sane) {
return res;
}
/// fill the in-memory buffer
switch (colr_.type()) {
case ibis::BYTE:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<signed char>*>(valr_), rtypes, rbuff,
*static_cast<array_t<signed char>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::UBYTE:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<unsigned char>*>(valr_), rtypes, rbuff,
*static_cast<array_t<unsigned char>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::SHORT:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<int16_t>*>(valr_), rtypes, rbuff,
*static_cast<array_t<int16_t>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::USHORT:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<uint16_t>*>(valr_), rtypes, rbuff,
*static_cast<array_t<uint16_t>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::INT:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<int32_t>*>(valr_), rtypes, rbuff,
*static_cast<array_t<int32_t>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::UINT:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<uint32_t>*>(valr_), rtypes, rbuff,
*static_cast<array_t<uint32_t>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::LONG:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<int64_t>*>(valr_), rtypes, rbuff,
*static_cast<array_t<int64_t>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::ULONG:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<uint64_t>*>(valr_), rtypes, rbuff,
*static_cast<array_t<uint64_t>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::FLOAT:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<float>*>(valr_), rtypes, rbuff,
*static_cast<array_t<float>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::DOUBLE:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<double>*>(valr_), rtypes, rbuff,
*static_cast<array_t<double>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
default:
LOGGER(ibis::gVerbose > 0)
<< "Warning -- " << evt << " can not handle join column of type "
<< ibis::TYPESTRING[(int)colr_.type()];
}
for (unsigned j = 0; j < cats.size(); ++ j) {
if (cats[j] != 0) {
ibis::bord::column *bc = dynamic_cast<ibis::bord::column*>
(static_cast<ibis::bord*>(res)->getColumn(j));
if (bc != 0)
bc->setDictionary(cats[j]);
}
}
return res;
} // ibis::jRange::select
int64_t ibis::jRange::count() const {
if (nrows >= 0) return nrows; // already have done this
if (maskr_.cnt() == 0 || masks_.cnt() == 0) {
return 0;
}
std::string mesg;
mesg = "jRange::count(";
mesg += desc_;
mesg += ")";
ibis::util::timer tm(mesg.c_str(), 1);
// allocate space for ordering arrays
orderr_ = new array_t<uint32_t>;
orders_ = new array_t<uint32_t>;
// Retrieve and sort the values
switch (colr_.type()) {
default:
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange[" << desc_
<< "] cann't handle join column of type " << colr_.type();
return -2;
case ibis::BYTE: {
valr_ = colr_.selectBytes(maskr_);
if (valr_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << colr_.name() << "->selectBytes("
<< maskr_.cnt() << ") failed";
return -3;
}
vals_ = cols_.selectBytes(masks_);
if (vals_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << cols_.name() << "->selectBytes("
<< masks_.cnt() << ") failed";
return -4;
}
nrows = ibis::util::sortMerge
(*static_cast<array_t<signed char>*>(valr_), *orderr_,
*static_cast<array_t<signed char>*>(vals_), *orders_,
delta1_, delta2_);
break;}
case ibis::UBYTE: {
valr_ = colr_.selectUBytes(maskr_);
if (valr_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << colr_.name() << "->selectUBytes("
<< maskr_.cnt() << ") failed";
return -3;
}
vals_ = cols_.selectUBytes(masks_);
if (vals_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << cols_.name() << "->selectUBytes("
<< masks_.cnt() << ") failed";
return -4;
}
nrows = ibis::util::sortMerge
(*static_cast<array_t<unsigned char>*>(valr_),
*orderr_,
*static_cast<array_t<unsigned char>*>(vals_),
*orders_, delta1_, delta2_);
break;}
case ibis::SHORT: {
valr_ = colr_.selectShorts(maskr_);
if (valr_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << colr_.name() << "->selectShorts("
<< maskr_.cnt() << ") failed";
return -3;
}
vals_ = cols_.selectShorts(masks_);
if (vals_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << cols_.name() << "->selectShorts("
<< masks_.cnt() << ") failed";
return -4;
}
nrows = ibis::util::sortMerge
(*static_cast<array_t<int16_t>*>(valr_), *orderr_,
*static_cast<array_t<int16_t>*>(vals_), *orders_,
delta1_, delta2_);
break;}
case ibis::USHORT: {
valr_ = colr_.selectUShorts(maskr_);
if (valr_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << colr_.name() << "->selectUShorts("
<< maskr_.cnt() << ") failed";
return -3;
}
vals_ = cols_.selectUShorts(masks_);
if (vals_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << cols_.name() << "->selectUShorts("
<< masks_.cnt() << ") failed";
return -4;
}
nrows = ibis::util::sortMerge
(*static_cast<array_t<uint16_t>*>(valr_), *orderr_,
*static_cast<array_t<uint16_t>*>(vals_), *orders_,
delta1_, delta2_);
break;}
case ibis::INT: {
valr_ = colr_.selectInts(maskr_);
if (valr_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << colr_.name() << "->selectInts("
<< maskr_.cnt() << ") failed";
return -3;
}
vals_ = cols_.selectInts(masks_);
if (vals_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << cols_.name() << "->selectInts("
<< masks_.cnt() << ") failed";
return -4;
}
nrows = ibis::util::sortMerge
(*static_cast<array_t<int32_t>*>(valr_), *orderr_,
*static_cast<array_t<int32_t>*>(vals_), *orders_,
delta1_, delta2_);
break;}
case ibis::UINT: {
valr_ = colr_.selectUInts(maskr_);
if (valr_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << colr_.name() << "->selectUInts("
<< maskr_.cnt() << ") failed";
return -3;
}
vals_ = cols_.selectUInts(masks_);
if (vals_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << cols_.name() << "->selectUInts("
<< masks_.cnt() << ") failed";
return -4;
}
nrows = ibis::util::sortMerge
(*static_cast<array_t<uint32_t>*>(valr_), *orderr_,
*static_cast<array_t<uint32_t>*>(vals_), *orders_,
delta1_, delta2_);
break;}
case ibis::LONG: {
valr_ = colr_.selectLongs(maskr_);
if (valr_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << colr_.name() << "->selectLongs("
<< maskr_.cnt() << ") failed";
return -3;
}
vals_ = cols_.selectLongs(masks_);
if (vals_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << cols_.name() << "->selectLongs("
<< masks_.cnt() << ") failed";
return -4;
}
nrows = ibis::util::sortMerge
(*static_cast<array_t<int64_t>*>(valr_), *orderr_,
*static_cast<array_t<int64_t>*>(vals_), *orders_,
delta1_, delta2_);
break;}
case ibis::ULONG: {
valr_ = colr_.selectULongs(maskr_);
if (valr_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << colr_.name() << "->selectULongs("
<< maskr_.cnt() << ") failed";
return -3;
}
vals_ = cols_.selectULongs(masks_);
if (vals_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << cols_.name() << "->selectULongs("
<< masks_.cnt() << ") failed";
return -4;
}
nrows = ibis::util::sortMerge
(*static_cast<array_t<uint64_t>*>(valr_), *orderr_,
*static_cast<array_t<uint64_t>*>(vals_), *orders_,
delta1_, delta2_);
break;}
case ibis::FLOAT: {
valr_ = colr_.selectFloats(maskr_);
if (valr_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << colr_.name() << "->selectFloats("
<< maskr_.cnt() << ") failed";
return -3;
}
vals_ = cols_.selectFloats(masks_);
if (vals_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << cols_.name() << "->selectFloats("
<< masks_.cnt() << ") failed";
return -4;
}
nrows = ibis::util::sortMerge
(*static_cast<array_t<float>*>(valr_), *orderr_,
*static_cast<array_t<float>*>(vals_), *orders_, delta1_, delta2_);
break;}
case ibis::DOUBLE: {
valr_ = colr_.selectDoubles(maskr_);
if (valr_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << colr_.name() << "->selectDoubles("
<< maskr_.cnt() << ") failed";
return -3;
}
vals_ = cols_.selectDoubles(masks_);
if (vals_ == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::count(" << desc_
<< ") call to " << cols_.name() << "->selectDoubles("
<< masks_.cnt() << ") failed";
return -4;
}
nrows = ibis::util::sortMerge
(*static_cast<array_t<double>*>(valr_), *orderr_,
*static_cast<array_t<double>*>(vals_), *orders_, delta1_, delta2_);
break;}
}
LOGGER(ibis::gVerbose > 2)
<< "jRange::count(" << desc_ << ") found " << nrows
<< " hit" << (nrows>1?"s":"");
return nrows;
} // ibis::jRange::count
/// Constructor.
ibis::jRange::jRange(const ibis::part& partr, const ibis::part& parts,
const ibis::column& colr, const ibis::column& cols,
double delta1, double delta2,
const ibis::qExpr* condr, const ibis::qExpr* conds,
const ibis::selectClause* sel, const ibis::fromClause* frm,
const char* desc)
: sel_(sel ? new ibis::selectClause(*sel) : 0),
frm_(frm ? new ibis::fromClause(*frm) : 0),
partr_(partr), parts_(parts), colr_(colr), cols_(cols),
delta1_(delta1), delta2_(delta2), orderr_(0), orders_(0),
valr_(0), vals_(0), nrows(-1) {
if (desc == 0 || *desc == 0) { // build a description string
std::ostringstream oss;
oss << "From " << partr.name() << " Join " << parts.name()
<< " On " << delta1 << " <= " << partr.name() << '.'
<< colr.name() << " - " << parts.name() << '.' << cols.name()
<< " <= " << delta2 << " Where ...";
desc_ = oss.str();
}
else {
desc_ = desc;
}
int ierr;
if (condr != 0) {
ibis::countQuery que(&partr);
ierr = que.setWhereClause(condr);
if (ierr < 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange(" << desc_ << ") could apply "
<< condr << " on partition " << partr.name()
<< ", ierr = " << ierr;
throw "jRange::ctor failed to apply conditions on partr"
IBIS_FILE_LINE;
}
ierr = que.evaluate();
if (ierr < 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange(" << desc_
<< ") could not evaluate " << que.getWhereClause()
<< " on partition " << partr.name() << ", ierr = " << ierr;
throw "jRange::ctor failed to evaluate constraints on partr"
IBIS_FILE_LINE;
}
maskr_.copy(*que.getHitVector());
}
else {
colr.getNullMask(maskr_);
}
if (conds != 0) {
ibis::countQuery que(&parts);
ierr = que.setWhereClause(conds);
if (ierr < 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange(" << desc_ << ") could apply "
<< conds << " on partition " << parts.name()
<< ", ierr = " << ierr;
throw "jRange::ctor failed to apply conditions on parts"
IBIS_FILE_LINE;
}
ierr = que.evaluate();
if (ierr < 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange(" << desc_
<< ") could not evaluate " << que.getWhereClause()
<< " on partition " << parts.name() << ", ierr = " << ierr;
throw "jRange::ctor failed to evaluate constraints on parts"
IBIS_FILE_LINE;
}
masks_.copy(*que.getHitVector());
}
else {
cols.getNullMask(masks_);
}
LOGGER(ibis::gVerbose > 2)
<< "jRange(" << desc_ << ") construction complete";
} // ibis::jRange::jRange
ibis::table*
ibis::jRange::fillResult(size_t nrows, double delta1, double delta2,
const std::string &desc,
const ibis::array_t<T>& rjcol,
const ibis::table::typeArray& rtypes,
const ibis::table::bufferArray& rbuff,
const ibis::array_t<T>& sjcol,
const ibis::table::typeArray& stypes,
const ibis::table::bufferArray& sbuff,
const ibis::table::stringArray& tcname,
const std::vector<uint32_t>& tcnpos) {
if (nrows > (rjcol.size() * sjcol.size()) ||
rtypes.size() != rbuff.size() || stypes.size() != sbuff.size() ||
tcname.size() != rtypes.size() + stypes.size() ||
tcnpos.size() != tcname.size()) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::fillResult can not proceed due "
"to invalid arguments";
return 0;
}
std::string tn = ibis::util::shortName(desc.c_str());
if (nrows == 0 || rjcol.empty() || sjcol.empty() ||
(stypes.empty() && rtypes.empty()))
return new ibis::tabula(tn.c_str(), desc.c_str(), nrows);
ibis::table::bufferArray tbuff(tcname.size());
ibis::table::typeArray ttypes(tcname.size());
IBIS_BLOCK_GUARD(ibis::table::freeBuffers, ibis::util::ref(tbuff),
ibis::util::ref(ttypes));
try {
// allocate enough space for the output table
for (size_t j = 0; j < tcname.size(); ++ j) {
if (tcnpos[j] < rtypes.size()) {
ttypes[j] = rtypes[tcnpos[j]];
tbuff[j] = ibis::table::allocateBuffer
(rtypes[tcnpos[j]], nrows);
}
else if (tcnpos[j] < rtypes.size()+stypes.size()) {
ttypes[j] = stypes[tcnpos[j]-rtypes.size()];
tbuff[j] = ibis::table::allocateBuffer
(stypes[tcnpos[j]-rtypes.size()], nrows);
}
else { // tcnpos is out of valid range
ttypes[j] = ibis::UNKNOWN_TYPE;
tbuff[j] = 0;
LOGGER(ibis::gVerbose > 0)
<< "Warning -- jRange::fillResult detects an "
"invalid tcnpos[" << j << "] = " << tcnpos[j]
<< ", should be less than " << rtypes.size()+stypes.size();
return 0;
}
}
}
catch (...) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- jRange::fillResult failed to allocate "
"sufficient memory for " << nrows << " row" << (nrows>1?"s":"")
<< " and " << rtypes.size()+stypes.size()
<< " column" << (rtypes.size()+stypes.size()>1?"s":"");
return 0;
}
size_t tind = 0; // row index into the resulting table
uint32_t ir0 = 0;
uint32_t ir1 = 0;
uint32_t is = 0;
const uint32_t nr = rjcol.size();
const uint32_t ns = sjcol.size();
while (ir0 < nr && is < ns) {
while (ir0 < nr && rjcol[ir0] < sjcol[is]+delta1)
++ ir0;
ir1 = (ir1>=ir0?ir1:ir0);
while (ir1 < nr && rjcol[ir1] <= sjcol[is]+delta2)
++ ir1;
if (ir1 > ir0) { // found matches
size_t is0 = is;
while (is < ns && sjcol[is] == sjcol[is0])
++ is;
LOGGER(ibis::gVerbose > 5)
<< "DEBUG -- jRange::fillResult: ir0=" << ir0 << ", ir1="
<< ir1 << ", is0=" << is0 << ", is1=" << is << ", rjcol["
<< ir0 << "]=" << rjcol[ir0] << ", rjcol[" << ir1 << "]="
<< rjcol[ir1] << ", sjcol[" << is0 << "]=" << sjcol[is0]
<< ", sjcol[" << is << "]=" << sjcol[is];
for (size_t jr = ir0; jr < ir1; ++ jr) {
for (size_t js = is0; js < is; ++ js) {
for (size_t jt = 0; jt < tcnpos.size(); ++ jt) {
if (tcnpos[jt] < rbuff.size()) {
ibis::bord::copyValue(rtypes[tcnpos[jt]],
tbuff[jt], tind,
rbuff[tcnpos[jt]], jr);
}
else {
ibis::bord::copyValue
(stypes[tcnpos[jt]-rtypes.size()],
tbuff[jt], tind,
sbuff[tcnpos[jt]-rtypes.size()], js);
}
} // jt
++ tind;
} // js
} // jr
}
else {
++ is;
}
} // while ...
if (tind != nrows) {
LOGGER(ibis::gVerbose >= 0)
<< "Warning -- jRange::fillResult expected to produce "
<< nrows << " row" << (nrows>1?"s":"") << ", but produced "
<< tind << " instead";
return 0;
}
LOGGER(ibis::gVerbose > 3)
<< "jRange(" << desc << ")::fillResult produced " << tind
<< " row" << (tind>1?"s":"") << " for \"" << typeid(T).name()
<< '[' << rjcol.size() << "] - " << typeid(T).name()
<< '[' << sjcol.size() << "] between " << delta1 << " and " << delta2
<< '\"';
return new ibis::bord(tn.c_str(), desc.c_str(), nrows,
tbuff, ttypes, tcname);
} // ibis::jRange::fillResult
return_type main(int argc, /*argument count*/
char* argv[] /*argument array*/
)
{
int err_no = ZERO;
char *sdp_msg_string = NULL;
sdp_session_t *my_session = NULL;
char buff[FILE_SIZE];
if(argc != ARGC)
{
error_handling(ERR_MAJOR, SDP_ARGC_ERROR,
"Usage : <executable> <file name>");
exit(EXIT_FAILURE);
}
memset(buff, 0, FILE_SIZE);
if(gethostname(buff, FILE_SIZE) != SUCCESS)
{
error_handling(ERR_MAJOR, SDP_GETHOSTNAME_ERROR,
"sdp_server_main : Gethostname error");
exit(EXIT_FAILURE);
}
printf("Hostname is : %s\n ", buff);
LOGGER(LOG_CRITICAL, "Start of sdp_hash_append");
if(sdp_hash_append(argv[1]) != SUCCESS)
{
error_handling(ERR_MAJOR, SDP_FILE_HASH_ERROR,
"sdp_server_main : sdp_hash_append error.");
LOGGER(LOG_CRITICAL, "End of sdp_hash_append");
exit(EXIT_FAILURE);
}
LOGGER(LOG_CRITICAL, "End of sdp_hash_append");
log_level = LOG_CRITICAL;
log_max = LOG_MAX;
if(NULL == strncpy(log_file,"sdp_server.log", strlen("sdp_server.log") + 1))
{
error_handling(ERR_MAJOR, SDP_STRNCPY_ERROR,
"sdp_server_main : strncpy error.");
exit(EXIT_FAILURE);
}
if(NULL == strncpy(program_name,"main_server.c", strlen("main_server.c") + 1))
{
error_handling(ERR_MAJOR, SDP_STRNCPY_ERROR,
"sdp_server_main : strncpy error.");
exit(EXIT_FAILURE);
}
LOGGER(LOG_CRITICAL, "Start of sdp_checker");
err_no=sdp_checker(inputfile);
if(err_no != SUCCESS)
{
error_handling(ERR_MAJOR, SDP_CHECKER_ERROR,
"sdp_server_main : sdp_checker error.");
LOGGER(LOG_CRITICAL, "End of sdp_checker");
exit(EXIT_FAILURE);
}
LOGGER(LOG_CRITICAL, "End of sdp_checker");
my_session = (sdp_session_t *)calloc(1, sizeof(sdp_session_t));
if(NULL == my_session)
{
error_handling(ERR_MAJOR, SDP_STRING_MALLOC_ERROR,
"sdp_server_main : malloc error.");
exit(EXIT_FAILURE);
}
LOGGER(LOG_CRITICAL, "Start of sdp_populate_message");
if(sdp_populate_message(my_session, inputfile) != SUCCESS)
{
error_handling(ERR_MAJOR, SDP_POPULATE_ERROR,
"sdp_server_main : sdp_populate_message error.");
LOGGER(LOG_CRITICAL, "End of sdp_populate_message");
myfree(my_session);
exit(EXIT_FAILURE);
}
LOGGER(LOG_CRITICAL, "End of sdp_populate_message");
LOGGER(LOG_CRITICAL, "Start of sdp_session_to_string");
sdp_msg_string = sdp_session_to_str(my_session, &err_no);
if(sdp_msg_string == NULL)
{
error_handling(ERR_MAJOR, SDP_SESSION_TO_STR_ERROR,
"sdp_server_main : sdp_session_to_str error.");
LOGGER(LOG_CRITICAL, "End of sdp_session_to_string");
myfree(my_session);
exit(EXIT_FAILURE);
}
LOGGER(LOG_CRITICAL, "End of sdp_session_to_string");
myfree(my_session);
printf("%s\n",sdp_msg_string);
LOGGER(LOG_CRITICAL, "Start of sdp_sender");
if(sdp_server_sender(sdp_msg_string) != SUCCESS)
{
error_handling(ERR_MAJOR, SDP_SENDER_ERROR,
"sdp_server_main : sdp_server_sender error.");
LOGGER(LOG_CRITICAL, "End of sdp_server_sender");
free(sdp_msg_string);
exit(EXIT_FAILURE);
}
LOGGER(LOG_CRITICAL, "End of sdp_server_sender");
free(sdp_msg_string);
return SUCCESS;
}
ibis::math::term* ibis::selectClause::addRecursive(ibis::math::term*& tm) {
if (tm == 0) return tm;
switch (tm->termType()) {
default:
case ibis::math::NUMBER:
case ibis::math::STRING:
break; // nothing to do
case ibis::math::VARIABLE: {
ibis::selectClause::variable *var =
dynamic_cast<ibis::selectClause::variable *>(tm);
if (var == 0) { // a bare variable
const char* vname =
static_cast<ibis::math::variable*>(tm)->variableName();
if (ordered_.find(vname) == ordered_.end()) {
const unsigned pos = atms_.size();
aggr_.push_back(ibis::selectClause::NIL_AGGR);
atms_.push_back(tm->dup());
ordered_[vname] = pos;
LOGGER(ibis::gVerbose > 5)
<< "selectClause::addRecursive -- adding term "
<< pos << ": " << vname;
}
}
break;}
case ibis::math::STDFUNCTION1:
case ibis::math::CUSTOMFUNCTION1:
case ibis::math::STRINGFUNCTION1: {
ibis::math::term *nxt =
reinterpret_cast<ibis::math::term*>(tm->getLeft());
if (nxt == 0) {
return nxt;
}
else if (hasAggregation(nxt)) {
ibis::math::term *tmp = addRecursive(nxt);
if (tmp != nxt)
tm->getLeft() = tmp;
}
else {
const unsigned pos = atms_.size();
aggr_.push_back(ibis::selectClause::NIL_AGGR);
atms_.push_back(tm);
LOGGER(ibis::gVerbose > 5)
<< "selectClause::addRecursive -- adding term "
<< pos << ": " << aggDescription(pos);
std::ostringstream oss;
oss << "__" << std::hex << pos;
ordered_[oss.str()] = pos;
return new ibis::selectClause::variable(oss.str().c_str(), this);
}
break;}
case ibis::math::OPERATOR:
case ibis::math::STDFUNCTION2:
case ibis::math::CUSTOMFUNCTION2:
case ibis::math::STRINGFUNCTION2: {
ibis::math::term *left =
reinterpret_cast<ibis::math::term*>(tm->getLeft());
ibis::math::term *right =
reinterpret_cast<ibis::math::term*>(tm->getRight());
if (left == 0) {
if (right == 0) {
return 0;
}
else if (dynamic_cast<ibis::selectClause::variable*>(right) == 0) {
tm->getRight() = addRecursive(right);
}
}
else if (dynamic_cast<ibis::selectClause::variable*>(left) != 0) {
if (dynamic_cast<ibis::selectClause::variable*>(right) == 0) {
tm->getRight() = addRecursive(right);
}
}
else if (dynamic_cast<ibis::selectClause::variable*>(right) != 0) {
tm->getLeft() = addRecursive(left);
}
else if (hasAggregation(tm)) {
tm->getLeft() = addRecursive(left);
tm->getRight() = addRecursive(right);
}
else {
const unsigned pos = atms_.size();
aggr_.push_back(ibis::selectClause::NIL_AGGR);
atms_.push_back(tm);
LOGGER(ibis::gVerbose > 5)
<< "selectClause::addRecursive -- adding term "
<< pos << ": " << aggDescription(pos);
std::ostringstream oss;
oss << "__" << std::hex << pos;
ordered_[oss.str()] = pos;
return new ibis::selectClause::variable(oss.str().c_str(), this);
}
break;}
}
return tm;
} // ibis::selectClause::addRecursive
void stub_call(int log_level,char* message)
{
LOGGER(log_level, message);
}
bool EflResources::copyResource( Evas_Object* const _image
, std::string const& _path
, bool const _keep_aspect
, int _width
, int _height ) const
{
bool result( true );
Evas_Object* object = nullptr;
if( 0 != preloaded_images__.count( _path ) )
object = preloaded_images__.find( _path )->second;
else
{
object = preloaded_images__.find( IMG_DIR "/placeholder.png" )->second;
LOGGER( "Could not find file among preloaded images: " + _path );
result = false;
}
int src_w = 0;
int src_h = 0;
evas_object_image_size_get( object
, &src_w
, &src_h );
evas_object_image_size_set( _image
, src_w
, src_h );
evas_object_image_alpha_set( _image
, evas_object_image_alpha_get( object ) );
evas_object_image_data_set( _image
, evas_object_image_data_get( object
, 0 ) );
if( _keep_aspect )
{
if( 0 == _width || 0 == _height )
{
evas_object_geometry_get( _image
, nullptr
, nullptr
, &_width
, &_height );
}
int new_w = 0;
int new_h = 0;
Utility::calculateImageSize( _width
, _height
, src_w
, src_h
, new_w
, new_h );
evas_object_resize( _image
, new_w
, new_h );
}
evas_object_image_pixels_dirty_set( _image
, 1 );
return result;
}
/**
* Accepts a connection and returns a socket
* for the recently accepted connection
*
* @param listen_socket socket in a 'listening' state
* @param accept_socket pointer by reference to store the accepted connection socket
*
* @return TRUE if connection was accepted, otherwise FALSE
*/
int socket_accept(SOCKET_T * listen_socket, SOCKET_T ** accept_socket) {
int socket_handle;
SOCKET_T* new_acc_socket;
struct sockaddr sa_client;
unsigned int sa_client_size = sizeof(sa_client);
char buffer[1024];
MUTEX_LOCK(listen_socket->mutex);
if ( listen_socket != NULL ) {
// Allocates space for the socket structure
new_acc_socket = malloc(sizeof( SOCKET_T ));
memset(new_acc_socket, 0x00, sizeof( SOCKET_T ));
// Accepts connection
socket_handle = accept(listen_socket->handle, (struct sockaddr *) &sa_client, &sa_client_size);
if (socket_handle == -1) {
if ( errno == EAGAIN || errno == EWOULDBLOCK ) {
// Returns invalid, no connections pending
free(new_acc_socket);
MUTEX_UNLOCK(listen_socket->mutex);
return FALSE;
} else {
sprintf(buffer, "accept failed with error: %d\n", errno);
LOGGER(__FUNCTION__, buffer);
free(new_acc_socket);
MUTEX_UNLOCK(listen_socket->mutex);
return FALSE;
}
} else {
// Stores the new socket handle in the structure
new_acc_socket->handle = socket_handle;
// Copies the structure to the output parameter
*accept_socket = new_acc_socket;
// Creates mutex for the new socket
(*accept_socket)->mutex = (MUTEX_T*)malloc(sizeof(struct _mutex_t) );
memset( (*accept_socket)->mutex, 0x00, sizeof(struct _mutex_t));
MUTEX_CREATE( &((*accept_socket)->mutex) );
MUTEX_UNLOCK(listen_socket->mutex);
return TRUE;
}
}
MUTEX_UNLOCK(listen_socket->mutex);
return FALSE;
}
int
offline_capture(FILE *fcapture) {
struct bpf_program bpf;
char errbuf[PCAP_ERRBUF_SIZE];
char filter[300];
char ports_str[256];
char **ports;
int r, n_ports;
pcap = pcap_fopen_offline(fcapture, errbuf);
if (!pcap) {
LOGGER(ERROR, "pcap: %s\n", errbuf);
return 1;
}
if(port) {
int i, n = 0 ;
ports = split_string(port, strlen(port), ",", 1, &n_ports);
if(n_ports > 10) {
LOGGER(ERROR, "it's unscientific to listen so many ports.\n", errbuf);
return 1;
}
n = snprintf(ports_str, 256, "tcp port %s", ports[0]);
for(i = 1; i < n_ports; i++) {
n += snprintf(ports_str + n, 256, " or tcp port %s", ports[i]);
}
split_string_free(ports, n_ports);
}
// Capture only TCP
if (global_options.server && n_ports) {
sprintf(filter, "host %s and (%s)", global_options.server, ports_str);
} else if (global_options.server && !n_ports) {
sprintf(filter, "host %s", global_options.server);
} else if (!global_options.server && n_ports) {
sprintf(filter, "(%s)", ports_str);
} else {
sprintf(filter, "tcp");
}
if (pcap_compile(pcap, &bpf, filter, 1, 0)) {
LOGGER(ERROR, "pcap: %s\n", pcap_geterr(pcap));
return 1;
}
if (pcap_setfilter(pcap, &bpf)) {
LOGGER(ERROR, "pcap: %s\n", pcap_geterr(pcap));
return 1;
}
// The -1 here stands for "infinity"
r = pcap_loop(pcap, -1, process_packet, (unsigned char *) pcap);
if (r == -1) {
LOGGER(ERROR, "pcap: %s\n", pcap_geterr(pcap));
return 1;
}
return 1;
}
/**
* Checks if a group of sockets is available for read/write
*
* Upon return the lists will be already updated, having removed de nodes correponding
* to sockets that were not available for the requested operations (read/write)
*
* @param timeout timeout, or 0 for blocking
* @param read_s pointer to a socket list to check for read availability, or NULL if does not apply
* @param write_s pointer to a socket list to check for write availability, or NULL if does not apply
*
* @return TRUE, o FALSE si la operacion dio error o timeout
*/
int socket_select_multiple(int timeout, list_t* read_s, list_t* write_s ) {
list_node_t * seeker = NULL;
list_node_t * remover = NULL;
SOCKET_T * ptr_socket = NULL;
fd_set* readfds = NULL;
fd_set* writefds = NULL;
struct timeval tval_timeout;
int lockstate;
char buffer[1024];
int res;
// configures timeout
tval_timeout.tv_sec = timeout;
tval_timeout.tv_usec = 0;
// Reserva memoria para los sets
readfds = (fd_set*)malloc(sizeof(fd_set));
writefds = (fd_set*)malloc(sizeof(fd_set));
// Initializes sets
FD_ZERO(readfds);
FD_ZERO(writefds);
// Iterates through the read sockets list
seeker = read_s->first;
while ( seeker != NULL) {
ptr_socket = seeker->content;
if (ptr_socket != NULL) {
// Puts a lock on the socket's mutex
MUTEX_LOCK(ptr_socket->mutex);
// Adds socket to the read set
FD_SET( ptr_socket->handle, readfds);
}
seeker = seeker->next;
}
// Iterates through the write sockets list
seeker = write_s->first;
while ( seeker != NULL) {
ptr_socket = seeker->content;
if (ptr_socket != NULL) {
// If it is not locked
lockstate = MUTEX_IS_LOCKED(ptr_socket->mutex);
if ( lockstate == FALSE ) {
// Puts a lock on the socket's mutex
MUTEX_LOCK(ptr_socket->mutex);
}
// Adds socket to the write set
FD_SET( ptr_socket->handle, writefds );
}
seeker = seeker->next;
}
// Calls the select function with the generated sets
res = select(0, readfds, writefds, NULL, &tval_timeout);
if (res == -1) {
sprintf(buffer, "select failed with error: %d\n", errno);
LOGGER(__FUNCTION__, buffer);
// Frees memory
free(readfds);
free(writefds);
return FALSE;
}
//
// Goes through the socket lists checking for the available ones and
// removing the ones that are not available
//
//
// Goes through the read list
//
seeker = read_s->first;
while ( seeker != NULL) {
ptr_socket = seeker->content;
if (ptr_socket != NULL) {
// If a socket is not available for reading
if ( !FD_ISSET( ptr_socket->handle, readfds) ) {
// Removes the socket from the list
remover = seeker;
seeker = seeker->next;
LIST_REMOVE(read_s, remover, NULL);
// Removes the lock on the mutex
MUTEX_UNLOCK(ptr_socket->mutex);
// Moves to the next
continue;
} else {
// Removes the lock on the mutex
MUTEX_UNLOCK(ptr_socket->mutex);
}
}
// Moves to the next
seeker = seeker->next;
}
//
// Goes through the write list
//
seeker = write_s->first;
while ( seeker != NULL) {
ptr_socket = seeker->content;
if (ptr_socket != NULL) {
// If it is not locked
lockstate = MUTEX_IS_LOCKED(ptr_socket->mutex);
if ( lockstate == FALSE ) {
// Puts a lock on the socket's mutex
MUTEX_LOCK(ptr_socket->mutex);
}
// If socket is not available for writing
if ( !FD_ISSET( ptr_socket->handle, writefds) ) {
// Removes the socket from the list
remover = seeker;
seeker = seeker->next;
LIST_REMOVE(write_s, remover, NULL);
// Removes the lock on the socket's mutex
MUTEX_UNLOCK(ptr_socket->mutex);
// Moves to the next
continue;
} else {
// Removes the lock on the socket's mutex
MUTEX_UNLOCK(ptr_socket->mutex);
}
}
// Moves to the next
seeker = seeker->next;
}
// Frees memory
free(readfds);
free(writefds);
return TRUE;
}
void *
capture(void *arg) {
struct bpf_program bpf;
char errbuf[PCAP_ERRBUF_SIZE];
char filter[300];
char ports_str[256];
char **ports;
int r, n_ports;
// Second argument 0 stands for non-promiscuous mode
pcap = pcap_open_live(global_options.interface, CAPTURE_LENGTH, 0, READ_TIMEOUT, errbuf);
if (!pcap) {
LOGGER(ERROR, "pcap: %s\n", errbuf);
return NULL;
}
if(port) {
int i, n = 0 ;
ports = split_string(port, strlen(port), ",", 1, &n_ports);
if(n_ports > 10) {
LOGGER(ERROR, "it's unscientific to listen so many ports.\n", errbuf);
return NULL;
}
n = snprintf(ports_str, 256, "tcp port %s", ports[0]);
for(i = 1; i < n_ports; i++) {
n += snprintf(ports_str + n, 256, " or tcp port %s", ports[i]);
}
split_string_free(ports, n_ports);
}
// Capture only TCP
if (global_options.server && n_ports) {
sprintf(filter, "host %s and (%s)", global_options.server, ports_str);
} else if (global_options.server && !n_ports) {
sprintf(filter, "host %s", global_options.server);
} else if (!global_options.server && n_ports) {
sprintf(filter, " (%s)", ports_str);
} else {
sprintf(filter, "tcp");
}
if (pcap_compile(pcap, &bpf, filter, 1, 0)) {
LOGGER(ERROR, "pcap: %s\n", pcap_geterr(pcap));
return NULL;
}
if (pcap_setfilter(pcap, &bpf)) {
LOGGER(ERROR, "pcap: %s\n", pcap_geterr(pcap));
return NULL;
}
// The -1 here stands for "infinity"
r = pcap_loop(pcap, -1, process_packet, (unsigned char *) pcap);
if (r == -1) {
LOGGER(ERROR, "pcap: %s\n", pcap_geterr(pcap));
return NULL;
}
return NULL;
}
/**
** \~french
* \brief Fonction principale de l'outil createNodata
* \details Tout est contenu dans cette fonction. Le "cropage" se fait grâce à la classe TiffNodataManager, et le tuilage / compression est géré par TiledTiffWriter
* \param[in] argc nombre de paramètres
* \param[in] argv tableau des paramètres
* \return code de retour, 0 en cas de succès, -1 sinon
** \~english
* \brief Main function for tool createNodata
* \details All instrcutions are in this function. the crop is handled by the class TiffNodataManager and TiledTiffWriter make image tiled and compressed.
* \param[in] argc parameters number
* \param[in] argv parameters array
* \return return code, 0 if success, -1 otherwise
*/
int main ( int argc, char **argv ) {
char* input = 0, *output = 0;
int tileWidth = 256, tileHeight = 256;
Compression::eCompression compression = Compression::NONE;
bool crop = false;
bool debugLogger=false;
/* Initialisation des Loggers */
Logger::setOutput ( STANDARD_OUTPUT_STREAM_FOR_ERRORS );
Accumulator* acc = new StreamAccumulator();
Logger::setAccumulator ( INFO , acc );
Logger::setAccumulator ( WARN , acc );
Logger::setAccumulator ( ERROR, acc );
Logger::setAccumulator ( FATAL, acc );
std::ostream &logw = LOGGER ( WARN );
logw.precision ( 16 );
logw.setf ( std::ios::fixed,std::ios::floatfield );
// Récupération des paramètres
for ( int i = 1; i < argc; i++ ) {
if ( !strcmp ( argv[i],"-crop" ) ) {
crop = true;
continue;
}
if ( argv[i][0] == '-' ) {
switch ( argv[i][1] ) {
case 'h': // help
usage();
exit ( 0 );
case 'd': // debug logs
debugLogger = true;
break;
case 'c': // compression
if ( ++i == argc ) { error ( "Error in -c option", -1 ); }
if ( strncmp ( argv[i], "none",4 ) == 0 || strncmp ( argv[i], "raw",3 ) == 0 ) {
compression = Compression::NONE;
} else if ( strncmp ( argv[i], "png",3 ) == 0 ) {
compression = Compression::PNG;
} else if ( strncmp ( argv[i], "jpg",3 ) == 0 ) {
compression = Compression::JPEG;
} else if ( strncmp ( argv[i], "lzw",3 ) == 0 ) {
compression = Compression::LZW;
} else if ( strncmp ( argv[i], "zip",3 ) == 0 ) {
compression = Compression::DEFLATE;
} else if ( strncmp ( argv[i], "pkb",3 ) == 0 ) {
compression = Compression::PACKBITS;
} else {
error ( "Unknown compression : " + string(argv[i]), -1 );
}
break;
case 't':
if ( i+2 >= argc ) { error("Error in -t option", -1 ); }
tileWidth = atoi ( argv[++i] );
tileHeight = atoi ( argv[++i] );
break;
default:
error ( "Unknown option : " + string(argv[i]) ,-1 );
}
} else {
if ( input == 0 ) input = argv[i];
else if ( output == 0 ) output = argv[i];
else { error ( "Argument must specify ONE input file and ONE output file", 2 ); }
}
}
if (debugLogger) {
// le niveau debug du logger est activé
Logger::setAccumulator ( DEBUG, acc);
std::ostream &logd = LOGGER ( DEBUG );
logd.precision ( 16 );
logd.setf ( std::ios::fixed,std::ios::floatfield );
}
if ( input == 0 || output == 0 ) {
error ("Argument must specify one input file and one output file", -1);
}
FileImageFactory FIF;
if (crop && compression != Compression::JPEG) {
LOGGER_WARN("Crop option is reserved for JPEG compression");
crop = false;
}
// For jpeg compression with crop option, we have to remove white pixel, to avoid empty bloc in data
if ( crop ) {
LOGGER_DEBUG ( "Open image to read" );
// On récupère les informations nécessaires pour appeler le nodata manager
FileImage* tmpSourceImage = FIF.createImageToRead(input);
int spp = tmpSourceImage->channels;
int bps = tmpSourceImage->getBitsPerSample();
SampleFormat::eSampleFormat sf = tmpSourceImage->getSampleFormat();
delete tmpSourceImage;
if ( bps == 8 && sf == SampleFormat::UINT ) {
TiffNodataManager<uint8_t> TNM ( spp, white, true, fastWhite,white );
if ( ! TNM.treatNodata ( input,input ) ) {
error ( "Unable to treat white pixels in this image : " + string(input), -1 );
}
} else {
LOGGER_WARN( "Crop option ignored (only for 8-bit integer images) for the image : " << input);
}
}
LOGGER_DEBUG ( "Open image to read" );
FileImage* sourceImage = FIF.createImageToRead(input);
if (sourceImage == NULL) {
error("Cannot read the source image", -1);
}
if (debugLogger) {
sourceImage->print();
}
Rok4ImageFactory R4IF;
Rok4Image* rok4Image = R4IF.createRok4ImageToWrite(
output, BoundingBox<double>(0.,0.,0.,0.), -1, -1, sourceImage->getWidth(), sourceImage->getHeight(), sourceImage->channels,
sourceImage->getSampleFormat(), sourceImage->getBitsPerSample(), sourceImage->getPhotometric(), compression,
tileWidth, tileHeight
);
rok4Image->setExtraSample(sourceImage->getExtraSample());
if (rok4Image == NULL) {
error("Cannot create the ROK4 image to write", -1);
}
if (debugLogger) {
rok4Image->print();
}
LOGGER_DEBUG ( "Write" );
if (rok4Image->writeImage(sourceImage, crop) < 0) {
error("Cannot write ROK4 image", -1);
}
LOGGER_DEBUG ( "Clean" );
// Nettoyage
delete acc;
delete sourceImage;
delete rok4Image;
return 0;
}
int main(int argc, char **argv)
{
std::string varPathStr;
std::string varNameStr1;
std::string varNameStr2;
std::string varNameStr3;
parseArgs(argc, argv);
std::vector<double> beginList;
std::vector<double> endList;
std::vector<double> strideList;
beginList.resize(dimension);
endList.resize(dimension);
strideList.resize(dimension);
if (datafile.empty() || condstring == 0 || dimension == 0 ) {
std::cerr << "Usage:\n" << *argv
<< " -f data-file-name"
<< " -q query-conditions-in-a-single-string"
<< " -x histogram-dimension"
<< " -y begin"
<< " -e end"
<< " -s stride"
<< " [-i index-file-name]"
<< " [-g log-file-name]"
<< " [-n name-of-variable]"
<< " [-p path-of-variable]"
<< " [-m file model [HDF5(default), H5PART, NETCDF, PNETCDF]"
<< " [-b use-boundingbox-data-selection]"
<< " [-v verboseness]"
<< " [-l mpi-subarray-length]"
//<< "\n e.g: ./histogram -f h5uc-data-index.h5 -q 'px < 0.3' -n y -p TimeStep2 -x 1\n"
<< "\n e.g: ./histogram -f h5uc-data.h5 -i indexfile -q 'px<0.3 && py>0' -x 2 -n py,pz;"
<< " -y '0,-0.5;' -s '0.1,0.02;' -e '1,0;' -p TimeStep2\n\n"
<< "\tFor More detailed usage description and examples, please see file GUIDE"
<< std::endl;
return -1;
}
#ifndef FQ_NOMPI
MPI_Init(&argc, &argv);
int mpi_size, mpi_rank;
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
#endif
ibis::gParameters().add(FQ_REPORT_STATISTIC, "true");
ibis::horometer totTimer;
totTimer.start();
FQ::FileFormat model = FQ::FQ_HDF5;
if (fileModel != 0) {
std::string format = fileModel;
if (format.compare("HDF5") == 0) {
model = FQ::FQ_HDF5;
}
else if (format.compare("H5PART") == 0) {
model = FQ::FQ_H5Part;
}
else if (format.compare("NETCDF") == 0) {
model = FQ::FQ_NetCDF;
}
else if (format.compare("PNETCDF") == 0) {
model = FQ::FQ_pnetCDF;
}
}
if (! indexfile.empty()) {
if (verboseness > 1)
std::cout << "DEBUG: using indexfile \"" << indexfile.c_str() << "\" ... \n";
}
if (varPath != 0) {
if (verboseness > 1) std::cout << "Debug: use variable path \"" << varPath << "\"\n";
varPathStr = varPath;
}
// std::cout << "varName:" << varName << " begin:" << begin << " end:" << end << " stride:" << stride << std::endl;
varName1 = strtok(varName, ",;");
if (dimension>1) varName2 = strtok(NULL, ",;");
if (dimension>2) varName3 = strtok(NULL, ",;");
begin1 = atof(strtok(begin, ",;"));
if (dimension>1) begin2 = atof(strtok(NULL, ",;"));
if (dimension>2) begin3 = atof(strtok(NULL, ",;"));
end1 = atof(strtok(end, ",;"));
if (dimension>1) end2 = atof(strtok(NULL, ",;"));
if (dimension>2) end3 = atof(strtok(NULL, ",;"));
stride1 = atof(strtok(stride, ",;"));
if (dimension>1) stride2 = atof(strtok(NULL, ",;"));
if (dimension>2) stride3 = atof(strtok(NULL, ",;"));
//std::cout << "varName is " << varName1 << ", " << varName2 << ", " << varName3 << std::endl;
//std::cout << "begin is " << begin1 << ", " << begin2 << ", " << begin3 << std::endl;
//std::cout << "end is " << end1 << ", " << end2 << ", " << end3 << std::endl;
//std::cout << "stride is " << stride1 << ", " << stride2 << ", " << stride3 << std::endl;
if (varName1!=0) varNameStr1 = varName1;
if (dimension>1 && varName2!=0) varNameStr2 = varName2;
if (dimension>2 && varName3!=0) varNameStr3 = varName3;
/*
if (mpi_rank==0) {
unsigned int dims1 = static_cast<uint32_t>(1+floor((end1-begin1)/stride1));
unsigned int dims2 = static_cast<uint32_t>(1+floor((end2-begin2)/stride2));
unsigned int dims3 = static_cast<uint32_t>(1+floor((end3-begin3)/stride3));
std::cout << "dims1 * dims2 * dims3 = " << dims1 << " * " << dims2 << " * " << dims3 << std::endl;
}
*/
if (logfile.str().empty() != true) {
#ifndef FQ_NOMPI
logfile << mpi_rank << ".log";
#endif
if (verboseness > 1) std::cout << "Debug: using logfile \"" << logfile.str().c_str() << "\"\n";
}
if (verboseness >1) {
std::cout << "open the file handler" << std::endl;
}
// open the named file
QueryProcessor* queryProcessor = new QueryProcessor(datafile, model, indexfile, verboseness, "", logfile.str().c_str()); // the file handler
if (queryProcessor->isValid() == false) {
if (verboseness > 0) {
std::cout << "ERROR: failed to initiate the QueryProcessor object for file \""
<< datafile.c_str() << "\" ...\n";
std::cout << "REPORT: failed to complete processing query" << std::endl;
}
delete(queryProcessor);
#ifndef FQ_NOMPI
MPI_Finalize();
#endif
return -1;
}
uint64_t hits = 0;
// getNumHits
ibis::horometer timer;
timer.start();
hits = queryProcessor->getNumHits(condstring, varPathStr, mpi_dim, mpi_len);
timer.stop();
if (verboseness > 1)
std::cout << "Debug: conditions \"" << condstring
<< "\" number of hits " << hits << std::endl;;
if (hits == 0) {
if (verboseness > 1) {
std::cout << "Warning -- No element is seleteced ==>"
<< " the rest of the test is skipped!" << std::endl;
}
if (verboseness > 0) {
#ifndef FQ_NOMPI
if (mpi_rank==0) {
#endif
std::cout << "REPORT: successfully completed processing query with "
<< hits << " hits" << std::endl;
#ifndef FQ_NOMPI
}
#endif
}
delete(queryProcessor);
#ifndef FQ_NOMPI
MPI_Finalize();
#endif
return hits;
}
// executeQuery
std::vector<uint64_t> coords;
std::vector<uint32_t> counts;
bool herr = true;
// if (mpi_rank==0) std::cout<<"histogram starting..."<<std::endl;
if (varPath != 0) {
//coords.reserve(hits*dims.size());
// hits1 = queryProcessor->executeQuery((char*)condstring, coords, varPathStr, FQ::POINTS_SELECTION, mpi_dim, mpi_len);
if (dimension==1) {
//counts.assign(static_cast<uint32_t>(1+floor(end1-begin1)/stride1), 0);
herr = queryProcessor->get1DHistogram
((char*) condstring, varNameStr1, varPathStr, begin1, end1, stride1,
counts, mpi_dim, mpi_len);
} else if (dimension==2) {
// if (mpi_rank==0) std::cout << "in 2Dhistogram" << std::endl;
//counts.assign(static_cast<uint32_t>(1+floor(end1-begin1)/stride1)*
// static_cast<uint32_t>(1+floor(end2-begin2)/stride2), 0);
herr = queryProcessor->get2DHistogram
((char*) condstring, varPathStr, varNameStr1, begin1, end1, stride1,
varNameStr2, begin2, end2, stride2,
counts, mpi_dim, mpi_len);
// if (mpi_rank==0) std::cout << "out 2Dhistogram" << std::endl;
} else if (dimension==3) {
herr = queryProcessor->get3DHistogram
((char*) condstring, varPathStr,
varNameStr1, begin1, end1, stride1,
varNameStr2, begin2, end2, stride2,
varNameStr3, begin3, end3, stride3,
counts, mpi_dim, mpi_len);
}
if (! herr) {
LOGGER(ibis::gVerbose >= 0)
<< *argv << " failed to compute the histogram";
return -2;
}
/************************/
/* verify part */
/************************/
if (verification) {
//verify the Histogram
//if (mpi_rank==0) std::cout << "starting verify the histogram..." << std::endl;
uint64_t len = 1;
if (len) {
//std::cout<<"Warning: May use too large memory. Can only check sum.\n";
} else {
/* double data[len];
//
std::vector<uint32_t> temp_counts;
temp_counts.assign(static_cast<uint32_t>(1+floor(end-begin)/stride), 0);
bool verr = true;
//std::cout << "starting getData...." << std::endl;
verr = queryProcessor->getData(varNameStr, &data[0], varPathStr);
#ifndef FQ_NOMPI
if (mpi_rank==0) {
#endif
if (len<=1000000) {
//std::cout << "getData success" << std::endl;
std::cout << "temp Histogram" << std::endl;
// copy from fasbit parth.cpp get1DHistogram
if (len != 0) {
for (uint32_t i = 0; i < len; ++ i) {
++ temp_counts[static_cast<uint32_t>((data[i] - begin) / stride)];
}
}
//
std::cout << "temp Histogram" << std::endl;
std::cout << "temp_counts.size is "<< temp_counts.size() << std::endl;
for (int i=0; i<temp_counts.size(); i++) {
std::cout << "[" << begin+i*stride << ", " << begin+(i+1)*stride << "]:\t" << temp_counts[i] << std::endl;
}
std::cout << "test Histogram" << std::endl;
std::cout << "counts.size is "<< counts.size() << std::endl;
for (int i=0; i<counts.size(); i++) {
std::cout << "[" << begin+i*stride << ", " << begin+(i+1)*stride << "]:\t" << counts[i] << std::endl;
}
// verify two histogram vectors
if (counts!=temp_counts) {
std::cout << "ERROR:Vector is not match.Histogram fail." << std::endl;
} else {
std::cout << "histogram success" << std::endl;
}
}
#ifndef FQ_NOMPI
}
#endif
*/ }
//unsigned int hits = 0 ;
//hits = queryProcessor->getNumHits(condstring, varPathStr, mpi_dim, mpi_len);
uint64_t hits1 = 0;
for (int i=0; i<counts.size(); i++) {
hits1 += counts[i];
}
if (hits1 != hits) {
std::cout<<"Error:\tcheck sum failed. Num of Hit is " << hits << ",and histogram number is " << hits1<<std::endl;
} else
std::cout<<"verification result is correct.\n";
}
std::fstream histogramFile;
#ifndef FQ_NOMPI
if (mpi_rank==0) {
#endif
if (dimension==1) {
std::fstream file;
//char fileName[100]="";
//char path[]="/global/homes/v/vidcina/fq/example/";
//fileName<<dimension<<"D"<<"histogram["<<begin1<<":"<<stride1<<":"<<end1<<"].out";
//sprintf(fileName, "%s%d%s%d%s%d%s%d%s", path, dimension, "Dhistogram[", begin1, ":", stride1, ":", end1, "].out");
//std::string temp="";
//temp.push_back(fileName.str());
std::ostringstream fileName;
fileName << hist_path << "_"<< dimension << "D" << "histogram["
<< begin1 << ":" << stride1 << ":" <<end1 << "].out";
std::string str = fileName.str();
const char* chr = str.c_str();
file.open(chr, std::ios::out);
if ( file.fail() ) {
std::cout << str << std::endl;
std::cout << "openFile fail" << std::endl;
} else {
for (int i=0; i<counts.size(); i++) {
file << begin1+i*stride1 << "\t" << begin1+(i+1)*stride1 << "\t" << counts[i] << std::endl;
}
}
//histogramFile.close();
} else if (dimension==2) {
std::cout << "2DHistogram "
<< "Variable1 "<< varName1 << " begin " << begin1
<< " to " << end1 <<" stride is " << stride1
<< "Variable2 "<< varName2 << " begin " << begin2
<< " to " << end2 <<" stride is " << stride2
<< std::endl ;
std::cout << "counts.size is "<< counts.size() << std::endl;
unsigned int imax = static_cast<uint32_t>(1+floor((end1-begin1)/stride1));
unsigned int jmax = static_cast<uint32_t>(1+floor((end2-begin2)/stride2));
for (unsigned int i=0; i<imax; i++) {
for (unsigned int j=0; j<jmax; j++) {
std::cout << "[" << begin1+i*stride1 << ", "
<< begin1+(i+1)*stride1 << "), ["
<< begin2+j*stride2 << ", "
<< begin2+(j+1)*stride2 << "):\t"
<< counts[i*jmax+j] << std::endl;
}
}
} else if (dimension==3) {
std::cout << "3DHistogram "
<< "Variable1 "<< varName1 << " begin " << begin1
<< " to " << end1 << " stride is " << stride1
<< "Variable2 "<< varName2 << " begin " << begin2
<< " to " << end2 << " stride is " << stride2
<< "Variable3 "<< varName3 << " begin " << begin3
<< " to " << end2 << " stride is " << stride3
<< std::endl ;
std::cout << "counts.size is "<< counts.size() << std::endl;
unsigned int imax = static_cast<uint32_t>
(1+floor((end1-begin1)/stride1));
unsigned int jmax = static_cast<uint32_t>
(1+floor((end2-begin2)/stride2));
unsigned int kmax = static_cast<uint32_t>
(1+floor((end3-begin3)/stride3));
#ifndef FQ_NOMPI
if (mpi_rank==0 && imax*jmax*kmax!=counts.size()) {
std::cout<<"ERROR: counts.size not match."<<std::endl;
delete(queryProcessor);
MPI_Finalize();
return 0;
}
#endif
for (unsigned int i=0; i<imax; i++) {
for (unsigned int j=0; j<jmax; j++) {
for (unsigned int k=0; k<kmax; k++) {
if (easyToShow) {
if (counts[i*jmax*kmax + j*kmax + k]!=0) {
std::cout << "[" << begin1+i*stride1 << ", "
<< begin1+(i+1)*stride1 << "), ["
<< begin2+j*stride2 << ", "
<< begin2+(j+1)*stride2 << "), ["
<< begin3+k*stride3 << ", "
<< begin3+(k+1)*stride3 << "):\t"
<< counts[i*jmax*kmax + j*kmax + k]
<< std::endl;
}
} else {
std::cout << "[" << begin1+i*stride1 << ", "
<< begin1+(i+1)*stride1 << "), ["
<< begin2+j*stride2 << ", "
<< begin2+(j+1)*stride2 << "), ["
<< begin3+k*stride3 << ", "
<< begin3+(k+1)*stride3 << "):\t"
<< counts[i*jmax*kmax + j*kmax + k]
<< std::endl;
}
}
}
}
std::cout << "successfuly printed histogram" << std::endl;
}
#ifndef FQ_NOMPI
}
#endif
// }//end else
}//end if(!varPath)
// MPI_Barrier(MPI_COMM_WORLD);
/*
if (hits != hits1)
{
std::cout << "Error -- number of hits does not match!" << std::endl;
std::cout << "REPORT: failed to complete processing query" << std::endl;
delete(queryProcessor);
#ifndef FQ_NOMPI
MPI_Finalize();
#endif
return -1;
}
*/
if (verboseness > 0) {
#ifndef FQ_NOMPI
if (mpi_rank==0) {
#endif
std::cout << "REPORT: successfully completed get1DHistogram with "
<< counts.size() << " histogram size" << std::endl;
#ifndef FQ_NOMPI
}
#endif
}
delete(queryProcessor);
#ifndef FQ_NOMPI
MPI_Finalize();
#endif
totTimer.stop();
LOGGER(FastQuery::reportTiming())
<< "Statistic\thistogram::totTimer\t"
<< totTimer.CPUTime() << "\t" << totTimer.realTime()
<< "\t";
return hits;
} // main
// This function reads the data file and records the locations of the values
// in bakMap
void ibis::bak::mapValues(const char* f, ibis::bak::bakMap& bmap) const {
if (col == 0) return;
horometer timer;
if (ibis::gVerbose > 4)
timer.start();
const unsigned prec = parsePrec(*col); // the precision of mapped value
std::string fnm; // name of the data file
dataFileName(fnm, f);
if (fnm.empty()) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- bak::mapValues failed to determine the data file "
"name from \"" << (f ? f : "") << '"';
return;
}
uint32_t nev;
ibis::bitvector mask;
col->getNullMask(mask);
if (col->partition() != 0)
nev = col->partition()->nRows();
else
nev = mask.size();
if (nev == 0) return;
// need to use different types of array_t for different columns
switch (col->type()) {
case ibis::TEXT:
case ibis::UINT: {// unsigned int
array_t<uint32_t> val;
if (! fnm.empty())
ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
col->getValuesArray(&val);
if (val.size() <= 0) {
col->logWarning("bak::mapValues", "unable to read %s",
fnm.c_str());
}
else {
bmap.clear();
nev = val.size();
if (nev > mask.size())
mask.adjustSize(nev, nev);
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nev ? iix[1] : nev);
for (uint32_t i = *iix; i < k; ++i) {
double key = ibis::util::coarsen(val[i], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(i, 1);
if (grn.min > val[i]) grn.min = val[i];
if (grn.max < val[i]) grn.max = val[i];
}
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0) grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nev) {
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nev) nind = 0;
} // while (nind)
}
break;}
case ibis::INT: {// signed int
array_t<int32_t> val;
if (! fnm.empty())
ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
col->getValuesArray(&val);
if (val.size() <= 0) {
col->logWarning("bak::mapValues", "unable to read %s",
fnm.c_str());
}
else {
nev = val.size();
if (nev > mask.size())
mask.adjustSize(nev, nev);
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nev ? iix[1] : nev);
for (uint32_t i = *iix; i < k; ++i) {
double key = ibis::util::coarsen(val[i], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(i, 1);
if (grn.min > val[i]) grn.min = val[i];
if (grn.max < val[i]) grn.max = val[i];
}
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nev) {
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nev) nind = 0;
} // while (nind)
}
break;}
case ibis::FLOAT: {// (4-byte) floating-point values
array_t<float> val;
if (! fnm.empty())
ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
col->getValuesArray(&val);
if (val.size() <= 0) {
col->logWarning("bak::mapValues", "unable to read %s",
fnm.c_str());
}
else {
nev = val.size();
if (nev > mask.size())
mask.adjustSize(nev, nev);
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nev ? iix[1] : nev);
for (uint32_t i = *iix; i < k; ++i) {
double key = ibis::util::coarsen(val[i], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(i, 1);
if (grn.min > val[i]) grn.min = val[i];
if (grn.max < val[i]) grn.max = val[i];
}
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nev) {
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nev) nind = 0;
} // while (nind)
}
break;}
case ibis::DOUBLE: {// (8-byte) floating-point values
array_t<double> val;
if (! fnm.empty())
ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
col->getValuesArray(&val);
if (val.size() <= 0) {
col->logWarning("bak::mapValues", "unable to read %s",
fnm.c_str());
}
else {
nev = val.size();
if (nev > mask.size())
mask.adjustSize(nev, nev);
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nev ? iix[1] : nev);
for (uint32_t i = *iix; i < k; ++i) {
double key = ibis::util::coarsen(val[i], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(i, 1);
if (grn.min > val[i]) grn.min = val[i];
if (grn.max < val[i]) grn.max = val[i];
}
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nev) {
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nev) nind = 0;
} // while (nind)
}
break;}
case ibis::CATEGORY: // no need for a separate index
col->logWarning("bak::mapValues", "no need for binning -- should have "
"a basic bitmap index already");
return;
default:
col->logWarning("bak::mapValues", "unable to create bins for "
"this type of column");
return;
}
// make sure all bit vectors are the same size
for (ibis::bak::bakMap::iterator it = bmap.begin();
it != bmap.end(); ++ it) {
(*it).second.loc->adjustSize(0, nev);
// (*it).second.loc->compress();
}
// write out the current content
if (ibis::gVerbose > 2) {
if (ibis::gVerbose > 4) {
timer.stop();
col->logMessage("bak::mapValues", "mapped %lu values to %lu "
"%u-digit number%s in %g sec(elapsed)",
static_cast<long unsigned>(nev),
static_cast<long unsigned>(bmap.size()),
prec, (bmap.size() > 1 ? "s" : ""),
timer.realTime());
}
else {
col->logMessage("bak::mapValues", "mapped %lu values to %lu "
"%lu-digit number%s",
static_cast<long unsigned>(nev),
static_cast<long unsigned>(bmap.size()), prec,
(bmap.size() > 1 ? "s" : ""));
}
if (ibis::gVerbose > 6) {
ibis::util::logger lg;
printMap(lg(), bmap);
}
}
} // ibis::bak::mapValues
void ImageStackDirectoryDatasource::writeToLog()
{
LOGGER("Datasource type: ImageStackDirectoryDatasource");
}
/**
* Thread function for processing an incoming message
*
* @param proc_data_arg data structure with the thread parameters
*/
void process_incoming_request_worker( void * proc_data_arg ) {
int brecv = 0;
int selectval = 0;
char * recbuf = NULL;
char * incoming_message = NULL;
int message_complete = FALSE;
unsigned long message_type = 0;
long total_bytes_received = 0;
long incoming_msg_len = 0;
long var_part_size = 0;
int header_complete = FALSE;
unsigned long exec_timeout = 0;
PROCESS_DATA_T * proc_data = ( PROCESS_DATA_T * ) proc_data_arg;
incoming_message = malloc(HEADER_LEN);
brecv = incoming_msg_len = HEADER_LEN;
// Allocates initial space for message
recbuf = malloc(sizeof(char) * HEADER_LEN + VAR_PART_MINIMUM_LEN);
// Update moment of next timeout
exec_timeout = (GetTickCount() + gl_timeout);
while (message_complete != TRUE && abort_processes == FALSE) {
// Evaluates if operation timed out and cancels
if (GetTickCount() > exec_timeout) {
LOGGER(__FUNCTION__, "ERROR: Message transfer operation timed out.");
goto END_PROCESS_INCOMING_REQUEST;
}
selectval = SOCKET_SELECT(S_TIMEOUT, proc_data->connection, S_READ);
if ( selectval == S_READ ) {
// Attempts to receive the message
if ( ! SOCKET_RECV(proc_data->connection, &recbuf, &brecv) ) {
// Produces error on fail
LOGGER(__FUNCTION__, "ERROR: A connection problem occurred while attemting to receive message." );
break;
}
if ( brecv > 0) {
total_bytes_received += brecv;
// Updates moment of next timeout
exec_timeout = GetTickCount() + gl_timeout;
// Checks if header was previously completed
if ( header_complete == TRUE ) {
// Copies the new message fragment
memcpy( &incoming_message[total_bytes_received-brecv], recbuf, brecv );
// If received message size has reached the total message size
if ( total_bytes_received == ( HEADER_LEN + var_part_size ) ) {
// Message completed
message_complete = TRUE;
break;
}
else {
int bytes_left = incoming_msg_len - total_bytes_received;
if ( bytes_left < CHUNK_SIZE ) {
brecv = bytes_left;
}
else {
brecv = CHUNK_SIZE;
}
}
}
// Checks if header is complete
else if ( total_bytes_received == HEADER_LEN ) {
// Copies the new fragment of message
memcpy( &incoming_message[0], recbuf, total_bytes_received );
// Validates header and gets message type
message_type = IS_VALID_HEADER( incoming_message, &var_part_size, ( FILE_SND_B + FILE_RCV_B + FILE_DEL_B ) );
if ( message_type > 0x00 ) {
header_complete = TRUE;
if ( var_part_size == 0 ) {
// If var part size is 0 fails
LOGGER(__FUNCTION__, "ERROR: Length of variable part cannot be 0." );
break;
}
if (var_part_size > CHUNK_SIZE) {
brecv = CHUNK_SIZE;
}
else {
brecv = var_part_size;
}
incoming_msg_len = HEADER_LEN + var_part_size;
incoming_message = realloc( incoming_message, incoming_msg_len );
recbuf = realloc( recbuf, incoming_msg_len );
}
else {
// If header is not valid fails
LOGGER(__FUNCTION__, "ERROR: The message does not have a valid header." );
break;
}
}
}
}
}
proc_data->received_message = incoming_message;
proc_data->received_msg_len = incoming_msg_len;
// Sends an ACK message to client
if ( ! process_outgoing_message(proc_data->connection, MESSAGE_ACK, strlen(MESSAGE_ACK)) ) {
LOGGER(__FUNCTION__, "ERROR: Acknowledgment message could not be sent.");
goto END_PROCESS_INCOMING_REQUEST;
}
// If it is a File Receive message
if ( message_type == FILE_RCV_B ) {
// Process the message
process_file_receive( proc_data );
}
// If it is a File Send message
if ( message_type == FILE_SND_B ) {
// Process the message
process_file_send( proc_data );
}
// If it is a File Delete message
if ( message_type == FILE_DEL_B ) {
// Process the message
process_file_delete( proc_data );
}
END_PROCESS_INCOMING_REQUEST:
// Cleanup
if (incoming_message != NULL) {
free(incoming_message);
}
if (recbuf != NULL) {
free(recbuf);
}
{
int proc_id = proc_data->process_id;
SOCKET_CLOSE(&(proc_data->connection));
free(processes[proc_id].proc_data);
processes[proc_id].proc_data = NULL;
processes[proc_id].is_active = FALSE;
}
return;
}
void Enqueue::visit(StmtMatmul& s)
{
if (_failure) return;
if (_printer) _printer->visit(s);
// some compute devices are single precision only
if ( ! OCLhacks::singleton().supportFP64(_memMgr.deviceNum()) &&
(PrecType::Double == s.precA() ||
PrecType::Double == s.precB() ||
PrecType::Double == s.precC()) )
{
_failure = true;
return;
}
if (s.isMATMUL())
{
// check if device supports the Evergreen matrix multiply
if (OCLhacks::singleton().supportEvergreen( _memMgr.deviceNum() ))
{
// do not allow modulo stream array subscripting for K dimension
const size_t KfromA = s.isTransposeA() ? s.heightA() : s.widthA();
const size_t KfromB = s.isTransposeB() ? s.widthB() : s.heightB();
if (KfromA != KfromB)
{
_failure = true;
return;
}
// ATI Evergreen matrix multiply
Evergreen::MatmulMM matmul( _memMgr.deviceNum() );
// exogenous parameters
matmul.setBatching(_vt.numTraces());
if (s.isSameDataA()) matmul.setSameDataMatrixA();
if (s.isSameDataB()) matmul.setSameDataMatrixB();
matmul.setGeneral(s.isGEMM());
matmul.setPrecision(s.precA(),
s.precB(),
s.precC());
matmul.setDimensions(s.heightC(), // M
s.widthC(), // N
KfromA); // K
matmul.setDataLayout(s.isTransposeA(),
s.isTransposeB());
const vector< size_t > params = _jitMemo.autotuneLookup(matmul);
if (params.empty())
{
_failure = true;
return;
}
matmul.setParams(params);
ArrayBuf* A
= s.astvarA()->isTraceVariable()
? _memMgr.arrayBuf(s.astvarA()->variable(), _vt)
: _memMgr.arrayBuf(s.astvarA(), _vt);
ArrayBuf* B
= s.astvarB()->isTraceVariable()
? _memMgr.arrayBuf(s.astvarB()->variable(), _vt)
: _memMgr.arrayBuf(s.astvarB(), _vt);
ArrayBuf* C
= s.astvarC()->isTraceVariable()
? _memMgr.arrayBuf(s.astvarC()->variable(), _vt)
: _memMgr.arrayBuf(s.astvarC(), _vt);
const double alpha = s.alpha();
const double beta = s.beta();
const string kernelName = matmul.kernelName();
stringstream ss;
ss << matmul;
const string kernelSource = ss.str();
OCLkernel ckernel( *_memMgr.computeDevice() );
ckernel.buildJIT(kernelName,
kernelSource);
if (! ckernel.isOk())
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "compile error: " << kernelName;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
if (! matmul.setArgs(ckernel, A, B, C, alpha, beta))
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "set arguments error: " << kernelName;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
const vector< size_t > globalDims = matmul.globalWorkItems();
const vector< size_t > localDims = matmul.localWorkItems();
for (size_t i = 0; i < globalDims.size(); i++)
{
ckernel << OCLWorkIndex(globalDims[i], localDims[i]);
if (! ckernel.statusOp())
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "set index space dimension error: " << i;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
}
*_memMgr.computeDevice() << ckernel;
if (! _memMgr.computeDevice()->statusOp())
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "enqueue error: " << kernelName;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
*_memMgr.computeDevice() << FLUSH;
if (! _memMgr.computeDevice()->statusOp())
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "wait error: " << kernelName;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
matmul.clearArgs();
}
else
{
// there are no other GPU matrix multiply implementations except
// "Evergreen" (which works on both ATI and NVIDIA), so fail
_failure = true;
}
}
else if (s.isMATVEC())
{
// check if device supports the Evergreen matrix multiply
if (OCLhacks::singleton().supportEvergreen( _memMgr.deviceNum() ))
{
// do not allow modulo stream array subscripting for N dimension
const size_t NfromA = s.isTransposeA() ? s.heightA() : s.widthA();
const size_t NfromB = s.widthB();
if (NfromA != NfromB)
{
_failure = true;
return;
}
// ATI Evergreen matrix-vector multiply
Evergreen::MatmulMV matvec( _memMgr.deviceNum() );
// exogenous parameters
matvec.setBatching(_vt.numTraces());
if (s.isSameDataA()) matvec.setSameDataMatrixA();
matvec.setGeneral(s.isGEMM());
matvec.setPrecision(s.precA(),
s.precB(),
s.precC());
matvec.setDimensions(s.widthC(), // M
NfromA); // N
matvec.setDataLayout(s.isTransposeA());
const vector< size_t > params = _jitMemo.autotuneLookup(matvec);
if (params.empty())
{
_failure = true;
return;
}
matvec.setParams(params);
ArrayBuf* A
= s.astvarA()->isTraceVariable()
? _memMgr.arrayBuf(s.astvarA()->variable(), _vt)
: _memMgr.arrayBuf(s.astvarA(), _vt);
ArrayBuf* B
= s.astvarB()->isTraceVariable()
? _memMgr.arrayBuf(s.astvarB()->variable(), _vt)
: _memMgr.arrayBuf(s.astvarB(), _vt);
ArrayBuf* C
= s.astvarC()->isTraceVariable()
? _memMgr.arrayBuf(s.astvarC()->variable(), _vt)
: _memMgr.arrayBuf(s.astvarC(), _vt);
const double alpha = s.alpha();
const double beta = s.beta();
const string kernelName = matvec.kernelName();
stringstream ss;
ss << matvec;
const string kernelSource = ss.str();
OCLkernel ckernel( *_memMgr.computeDevice() );
ckernel.buildJIT(kernelName,
kernelSource);
if (! ckernel.isOk())
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "compile error: " << kernelName;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
if (! matvec.setArgs(ckernel, A, B, C, alpha, beta))
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "set arguments error: " << kernelName;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
const vector< size_t > globalDims = matvec.globalWorkItems();
const vector< size_t > localDims = matvec.localWorkItems();
for (size_t i = 0; i < globalDims.size(); i++)
{
ckernel << OCLWorkIndex(globalDims[i], localDims[i]);
if (! ckernel.statusOp())
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "set index space dimension error: " << i;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
}
*_memMgr.computeDevice() << ckernel;
if (! _memMgr.computeDevice()->statusOp())
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "enqueue error: " << kernelName;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
*_memMgr.computeDevice() << FLUSH;
if (! _memMgr.computeDevice()->statusOp())
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "wait error: " << kernelName;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
matvec.clearArgs();
}
else
{
// there are no other GPU matrix multiply implementations except
// "Evergreen" (which works on both ATI and NVIDIA), so fail
_failure = true;
}
}
else if (s.isVECMAT())
{
// check if device supports the Evergreen matrix multiply
if (OCLhacks::singleton().supportEvergreen( _memMgr.deviceNum() ))
{
// do not allow modulo stream array subscripting for N dimension
const size_t NfromA = s.widthA();
const size_t NfromB = s.isTransposeB() ? s.widthB() : s.heightB();
if (NfromA != NfromB)
{
_failure = true;
return;
}
// ATI Evergreen matrix-vector multiply
Evergreen::MatmulMV matvec( _memMgr.deviceNum() );
// exogenous parameters
matvec.setBatching(_vt.numTraces());
if (s.isSameDataB()) matvec.setSameDataMatrixA();
matvec.setGeneral(s.isGEMM());
matvec.setPrecision(s.precB(),
s.precA(),
s.precC());
matvec.setDimensions(s.widthC(), // M
NfromB); // N
matvec.setDataLayout(! s.isTransposeB());
const vector< size_t > params = _jitMemo.autotuneLookup(matvec);
if (params.empty())
{
_failure = true;
return;
}
matvec.setParams(params);
ArrayBuf* A
= s.astvarA()->isTraceVariable()
? _memMgr.arrayBuf(s.astvarA()->variable(), _vt)
: _memMgr.arrayBuf(s.astvarA(), _vt);
ArrayBuf* B
= s.astvarB()->isTraceVariable()
? _memMgr.arrayBuf(s.astvarB()->variable(), _vt)
: _memMgr.arrayBuf(s.astvarB(), _vt);
ArrayBuf* C
= s.astvarC()->isTraceVariable()
? _memMgr.arrayBuf(s.astvarC()->variable(), _vt)
: _memMgr.arrayBuf(s.astvarC(), _vt);
const double alpha = s.alpha();
const double beta = s.beta();
const string kernelName = matvec.kernelName();
stringstream ss;
ss << matvec;
const string kernelSource = ss.str();
OCLkernel ckernel( *_memMgr.computeDevice() );
ckernel.buildJIT(kernelName,
kernelSource);
if (! ckernel.isOk())
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "compile error: " << kernelName;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
if (! matvec.setArgs(ckernel, B, A, C, alpha, beta))
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "set arguments error: " << kernelName;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
const vector< size_t > globalDims = matvec.globalWorkItems();
const vector< size_t > localDims = matvec.localWorkItems();
for (size_t i = 0; i < globalDims.size(); i++)
{
ckernel << OCLWorkIndex(globalDims[i], localDims[i]);
if (! ckernel.statusOp())
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "set index space dimension error: " << i;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
}
*_memMgr.computeDevice() << ckernel;
if (! _memMgr.computeDevice()->statusOp())
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "enqueue error: " << kernelName;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
*_memMgr.computeDevice() << FLUSH;
if (! _memMgr.computeDevice()->statusOp())
{
#ifdef __LOGGING_ENABLED__
stringstream ss;
ss << "wait error: " << kernelName;
LOGGER(ss.str())
#endif
_failure = true;
return;
}
matvec.clearArgs();
}
else
{
// there are no other GPU matrix multiply implementations except
// "Evergreen" (which works on both ATI and NVIDIA), so fail
_failure = true;
}
}
}
/**
* Processes a File Send message from the client,
* performs and finalizes the operation
*
* @param proc_data_arg data structure with connection parameters
* and received message
*/
void process_file_send( PROCESS_DATA_T * proc_data ) {
const int NOT_FOUND = -1;
char * filename = NULL;
char * content = NULL;
char * request = NULL;
char * response = NULL;
int search_path_res = 0;
int search_length_res = 0;
int search_content_res = 0;
unsigned long response_len = 0;
char destination_dir[2048];
char l_msg[4096];
int result = RESULT_UNDEFINED;
unsigned long content_len = 0;
int param_path_len = 0;//= (proc_data->received_msg_len - HEADER_LEN - strlen(PARAM_PATH));
// Copies the request to a safe buffer for searching
request = (char*)malloc(sizeof(char) * proc_data->received_msg_len + 1);
memcpy(request, proc_data->received_message, proc_data->received_msg_len);
request[proc_data->received_msg_len] = '\0';
// Finds parameters positions
search_path_res = STR_SEARCH( request, PARAM_PATH, HEADER_LEN);
search_length_res = STR_SEARCH( request, PARAM_LENGTH, HEADER_LEN);
search_content_res = STR_SEARCH( proc_data->received_message, PARAM_CONTENT, HEADER_LEN);
if (search_path_res == NOT_FOUND || search_length_res == NOT_FOUND || search_content_res == NOT_FOUND) {
result = RESULT_INVALID_REQUEST;
goto END_PROCESS_FILE_SEND;
}
param_path_len = ( search_length_res - ( HEADER_LEN + strlen(MSG_SEPARATOR) + strlen(PARAM_PATH) ) );
// Gets name of file to receive
filename = (char*)malloc(sizeof(char) * param_path_len + 1);
memcpy(filename, &request[search_path_res + strlen(PARAM_PATH)], param_path_len);
filename[param_path_len] = '\0';
sprintf(l_msg, "A request has been received to receive the file: %s", filename);
LOGGER(__FUNCTION__, l_msg);
//
// Gets content length
//
content_len = atoi( &( proc_data->received_message[ ( search_length_res + strlen(PARAM_LENGTH) ) ] ) );
if (content_len == 0) {
result = RESULT_INVALID_REQUEST;
goto END_PROCESS_FILE_SEND;
}
//
// Gets content
//
content = (char*)malloc(sizeof(char) * content_len + 1);
memcpy(content, &(proc_data->received_message[ ( search_content_res + strlen(PARAM_CONTENT) ) ]), content_len);
content[content_len] = '\0';
// Makes a backup copy if file exists
if (file_exists(filename) == TRUE) {
char new_name[2048];
sprintf(new_name, "%s.bkp", filename);
if ( ! file_copy(filename, new_name, TRUE) ) {
sprintf(l_msg, "ERROR: Could not make backup copy of file (%s).", filename);
LOGGER(__FUNCTION__, l_msg);
}
}
// Prepares directories
file_get_base_path(filename, destination_dir);
if (destination_dir != NULL)
{
// If destination directory is not root
if (strcmp(destination_dir, ROOT_DIR) != 0 ) {
if ( ! file_directory_exists(destination_dir)) {
if ( ! file_mkdir_parent(destination_dir) ) {
result = RESULT_COULD_NOT_CREATE_DESTINATION_DIRECTORY;
goto END_PROCESS_FILE_SEND;
}
}
}
}
else {
sprintf(l_msg, "ERROR: The directory specified for the file %s is not valid.", filename);
LOGGER(__FUNCTION__, l_msg);
result = RESULT_INVALID_DESTINATION_DIRECTORY;
goto END_PROCESS_FILE_SEND;
}
//
// Writes base64 to a file, then decodes and unpacks file
//
{
char b64_filename[2048];
char gz_filename[2048];
FILE * f = NULL;
sprintf(b64_filename, "%s.b64", filename);
sprintf(gz_filename, "%s.gz", filename);
f = fopen(b64_filename, "w");
if (f != NULL) {
fwrite( content, sizeof(char), content_len, f );
fclose(f);
if ( base64_process_file( 'd', b64_filename, gz_filename, content_len ) ) {
if ( gz_unpack_file(gz_filename, filename) ) {
sprintf(l_msg, "file was succesfully unpacked (%s).", gz_filename);
LOGGER( __FUNCTION__, l_msg );
result = RESULT_SUCCESS;
}
else {
sprintf(l_msg, "ERROR: could not unpack file (%s).", gz_filename);
LOGGER( __FUNCTION__, l_msg );
result = RESULT_FILE_DECOMPRESS_ERROR;
}
}
else {
sprintf(l_msg, "ERROR: could not decode file (%s).", b64_filename);
LOGGER( __FUNCTION__, l_msg );
result = RESULT_FILE_DECODE_ERROR;
}
// Delete generated temporary files
if ( file_exists(b64_filename) ) {
remove(b64_filename);
}
if ( file_exists(gz_filename) ) {
remove(gz_filename);
}
}
}
END_PROCESS_FILE_SEND:
// Generates a response message
response = message_file_send_response( result, &response_len );
// Sends the response
if ( !process_outgoing_message( proc_data->connection, response, response_len ) ) {
LOGGER(__FUNCTION__, "File Receive operation response message could not be sent.");
}
// Cleanup
if (request != NULL) {
free(request);
}
if (response != NULL) {
free(response);
}
if (filename != NULL) {
free(filename);
}
if (content != NULL) {
free(content);
}
return;
}
/// main.
int main(int argc, char** argv) {
uint32_t maxrow=0, nrpd=0;
int nparts, ndigits, ierr;
// must have the output directory name
if (argc < 2) {
std::cerr << "\nUsage:\n" << *argv
<< " <output-dir> [#rows [#rows-per-dir [conf-file]]]\n"
<< "If the 4th argument is not provided, the number of "
"rows per directory will be determined by the memory cache size, "
"which is by default 1/2 of the physical memory size.\n"
<< std::endl;
return -1;
}
//ibis::gVerbose = 8;
// initialize the file manage with the 5th argument
ibis::init(argc>4 ? argv[4] : (const char*)0);
ibis::util::timer mytimer(*argv, 0);
if (argc > 2) // user specified maxrow
maxrow = (uint32_t)atof(argv[2]);
if (maxrow <= 0) {
double tmp = ibis::fileManager::currentCacheSize();
maxrow = (uint32_t)
ibis::util::compactValue(tmp / 120.0, tmp / 80.0);
nrpd = maxrow;
}
if (maxrow < 10)
maxrow = 10;
if (argc > 3) // user specified nrpd
nrpd = (uint32_t) atof(argv[3]);
if (nrpd <= 0) {
double tmp = ibis::fileManager::currentCacheSize();
nrpd = (uint32_t)
ibis::util::compactValue(tmp / 120.0, tmp / 80.0);
}
if (nrpd > maxrow) nrpd = maxrow;
ibis::table::row val;
std::auto_ptr<ibis::tablex> tab(ibis::tablex::create());
initColumns(*tab, val);
ierr = tab->reserveBuffer(nrpd);
if (ierr > 0 && (unsigned)ierr < nrpd)
nrpd = ierr;
LOGGER(1) << *argv << ' ' << argv[1] << ' ' << maxrow << ' ' << nrpd
<< std::endl;
nparts = maxrow / nrpd;
nparts += (maxrow > nparts*nrpd);
ierr = nparts;
for (ndigits = 1, ierr >>= 4; ierr > 0; ierr >>= 4, ++ ndigits);
for (uint32_t irow = 1; irow <= maxrow;) {
const uint32_t end = irow - 1 + nrpd;
TDList tdl;
std::string dir = argv[1];
if (nparts > 1) { // figure out the directory name
const char* str = strrchr(argv[1], FASTBIT_DIRSEP);
if (str != 0) {
if (str[1] == 0) {
while (str-1 > argv[1]) {
if (*(str-1) == FASTBIT_DIRSEP) break;
else -- str;
}
}
else {
++ str;
}
}
std::ostringstream oss;
oss << FASTBIT_DIRSEP << (str ? str : "_") << std::hex
<< std::setprecision(ndigits) << std::setw(ndigits)
<< std::setfill('0') << irow / nrpd;
dir += oss.str();
}
for (; irow <= end; ++ irow) {
fillRow(irow, val, tdl);
ierr = tab->appendRow(val);
LOGGER(ierr != 6)
<< "Warning -- " << *argv << " failed to append row " << irow
<< " to the in-memory table, appendRow returned " << ierr;
LOGGER(irow % 100000 == 0) << " . " << irow;
}
LOGGER(1) << "\n";
ierr = tab->write(dir.c_str());
LOGGER(ierr < 0)
<< "Warning -- " << *argv << " failed to write " << tab->mRows()
<< " rows to " << dir << ", ibis::tablex::write returned " << ierr;
writeTDList(tdl, dir.c_str());
tab->clearData();
tdl.clear();
}
return 0;
} // main
/**
* Sends a synchronous message through a connected node
*
* @param connection conexion on which the message will be sent
* @param outgoing_message outgoing message
* @param outgoing_message_len outgoing message length
*
* @return TRUE if message could be sent, otherwise FALSE
*/
int process_outgoing_message( SOCKET_T * connection, char * outgoing_message, int outgoing_message_len ) {
int bsent = 0;
int selectval = 0;
long total_bytes_sent = 0;
long total_bytes_left = 0;
long chunk_size = HEADER_LEN;
int send_error = FALSE;
int timed_out = FALSE;
int message_send_success = FALSE;
unsigned long exec_timeout = 0;
char * progress;
// Updates moment for next timeout
exec_timeout = GetTickCount() + gl_timeout;
// Send message loop
while ( selectval != S_WRITE && abort_processes == FALSE ) {
// If operation timed out cancel
if (GetTickCount() > exec_timeout) {
timed_out = TRUE;
goto END_PROCESS_OUTGOING_MESSAGE;
}
// Once the header was sent sends the rest of the message...
while (total_bytes_sent < outgoing_message_len && abort_processes == FALSE) {
// If operation timed out cancel
if (GetTickCount() > exec_timeout) {
timed_out = TRUE;
goto END_PROCESS_OUTGOING_MESSAGE;
}
// ...but first it sends only the header
while (total_bytes_sent < HEADER_LEN && abort_processes == FALSE) {
// If operation timed out cancel
if (GetTickCount() > exec_timeout) {
timed_out = TRUE;
goto END_PROCESS_OUTGOING_MESSAGE;
}
selectval = SOCKET_SELECT(S_TIMEOUT, connection, S_WRITE);
if ( selectval == S_WRITE ) {
// Attempts to send the message
if ( ! SOCKET_SEND(connection, outgoing_message, chunk_size, &bsent) ) {
// Produces error on fail
send_error = TRUE;
break;
}
total_bytes_sent += bsent;
}
}
// If message is only a header then it is complete
if (total_bytes_sent == outgoing_message_len) {
break;
}
progress = &outgoing_message[total_bytes_sent];
// Updates bytes left to send count
total_bytes_left = outgoing_message_len - total_bytes_sent;
if (total_bytes_left > CHUNK_SIZE) {
chunk_size = CHUNK_SIZE;
}
else {
chunk_size = total_bytes_left;
}
selectval = SOCKET_SELECT(S_TIMEOUT, connection, S_WRITE);
if ( selectval == S_WRITE ) {
// Attempts to send the message following the header
if ( ! SOCKET_SEND(connection, progress, chunk_size, &bsent) ) {
// Produces error on fail
send_error = TRUE;
break;
}
total_bytes_sent += bsent;
}
}
// If bytes sent has reached total message size
if ( total_bytes_sent == outgoing_message_len ) {
message_send_success = TRUE;
break;
}
}
END_PROCESS_OUTGOING_MESSAGE:
// If an error occurred or operation timed out
if ( send_error == TRUE || timed_out) {
char log[_BUFFER_SIZE_L];
memset(log, 0x00, _BUFFER_SIZE_L);
strcpy( log, "Failed while attempting to send the following message: ");
if (outgoing_message_len >= _BUFFER_SIZE_L) {
memcpy(log, outgoing_message, _BUFFER_SIZE_L - 1);
}
else {
memcpy(log, outgoing_message, outgoing_message_len);
}
LOGGER(__FUNCTION__, log);
sprintf( log, "[Number of bytes sent:%ld]", total_bytes_sent );
LOGGER(__FUNCTION__, log);
}
return message_send_success;
}
void ProgressLogger::printToConsole(std::string text)
{
LOGGER(text);
}