Пример #1
0
/// Record an aggregation function.  Return a math term of the type
/// variable to the caller so the caller can continue to build up a larger
/// expression.  For simplicity, the variable name is simply "__hhh", where
/// "hhh" is the size of aggr_ in hexadecimal.
///
/// @note This function takes charge of expr.  It will free the object if
/// the object is not passed on to other operations.  This can happen when
/// the particular variable appeared already in the select clause.
ibis::math::variable*
ibis::selectClause::addAgregado(ibis::selectClause::AGREGADO agr,
                                ibis::math::term *expr) {
    if (agr != ibis::selectClause::NIL_AGGR &&
        hasAggregation(expr)) {
        LOGGER(ibis::gVerbose >= 0)
            << "Warning -- selectClause can not have aggregations inside "
            "another aggregation operation (" << *expr << ')';
        throw "selectClause::addAgregado failed due to nested aggregations"
            IBIS_FILE_LINE;
    }

    const unsigned end = atms_.size();
    LOGGER(ibis::gVerbose > 5)
        << "selectClause::addAgregado  -- adding term " << end << ": "
        << aggDescription(agr, expr);
    if (expr->termType() != ibis::math::VARIABLE) {
        aggr_.push_back(agr);
        atms_.push_back(expr);
        std::ostringstream oss;
        oss << "__" << std::hex << end;
        ordered_[oss.str()] = end;
        return new ibis::selectClause::variable(oss.str().c_str(), this);
    }
    else {
        ibis::math::variable *var = static_cast<ibis::math::variable*>(expr);
        ibis::selectClause::StringToInt::const_iterator it =
            ordered_.find(var->variableName());
        if (it == ordered_.end()) { // no in the existing list
            aggr_.push_back(agr);
            atms_.push_back(expr);
            ordered_[var->variableName()] = end;
            if (agr != ibis::selectClause::NIL_AGGR) {
                std::ostringstream oss;
                oss << "__" << std::hex << end;
                ordered_[oss.str()] = end;
                return new ibis::selectClause::variable
                    (oss.str().c_str(), this);
            }
            else {
                return var->dup();
            }
        }
        else if (agr != aggr_[it->second]) { // new aggregation
            aggr_.push_back(agr);
            atms_.push_back(expr);
            if (agr != ibis::selectClause::NIL_AGGR) {
                std::ostringstream oss;
                oss << "__" << std::hex << end;
                ordered_[oss.str()] = end;
                return new ibis::selectClause::variable
                    (oss.str().c_str(), this);
            }
            else {
                ordered_[var->variableName()] = end;
                return var->dup();
            }
        }
        else { // the variable has appeared before
            delete expr;
            std::ostringstream oss;
            oss << "__" << std::hex << it->second;
            return new ibis::selectClause::variable(oss.str().c_str(), this);
        }
    }
    return 0;
} // ibis::selectClause::addAgregado
Пример #2
0
/**
 * Processes a File Receive message from the client, 
 * performs and finalizes the operation
 *
 * @param proc_data_arg           data structure with connection parameters 
 *                                and received message
 */
void process_file_receive( PROCESS_DATA_T * proc_data ) {

  const int NOT_FOUND = -1;

  char * filename   = NULL;
  char l_msg[4096];

  char * request    = NULL;
  char * response   = NULL;

  int search_filename_res    = 0;
  unsigned long response_len = 0;

  int result = RESULT_UNDEFINED;
  const int param_len = (proc_data->received_msg_len - HEADER_LEN - strlen(PARAM_FILENAME));

  // Copies the request to a safe buffer for searching
  request = (char*)malloc(sizeof(char) * proc_data->received_msg_len + 1);
  memcpy(request, proc_data->received_message, proc_data->received_msg_len);
  request[proc_data->received_msg_len] = '\0';

  search_filename_res = STR_SEARCH( request, PARAM_FILENAME, HEADER_LEN);
  if ( search_filename_res == NOT_FOUND ) {

    result = RESULT_INVALID_REQUEST;
    goto END_PROCESS_FILE_RECEIVE;
  }
  
  // Gets name of file to send
  filename = (char*)malloc(sizeof(char) * proc_data->received_msg_len);
  memcpy( filename, &(request[ ( search_filename_res + strlen(PARAM_FILENAME) ) ]), param_len );
  filename[param_len] = '\0';

  sprintf(l_msg, "A request has been received to send the following file: %s", filename);
  LOGGER(__FUNCTION__, l_msg);

  //
  // Find, pack, and encode file
  //
  {
    long long filesize = file_size(filename);

    if ( ! file_exists(filename) || filesize <= 0 ) {

      LOGGER(__FUNCTION__, "No files have been found for the specified mask.");
      result = RESULT_FILE_NOT_FOUND;
    } else {

      char gzip_output[256];
      char b64_output[256];

      sprintf(gzip_output, ".\\%d-%lu.gz", proc_data->process_id, GetTickCount() );
      sprintf(b64_output, ".\\%d-%lu.b64", proc_data->process_id, GetTickCount() );

      if ( gz_pack_file(filename, gzip_output) == TRUE )
      {

        if ( base64_process_file('e', gzip_output, b64_output, filesize) == TRUE )
        {
          unsigned long bytes_read;
          char * buffer = NULL;
          FILE * f = NULL;

          filesize = file_size(b64_output);
          buffer = (char*)malloc(sizeof(char) * filesize + 1);
          memset(buffer, 0x00, filesize + 1);

          f = fopen(b64_output, "r");
          if (f != NULL)
          {
            bytes_read = fread(buffer, 1, filesize, f);

            sprintf(l_msg, "%lu bytes read from file %s to process and send.", bytes_read, b64_output);
            LOGGER(__FUNCTION__, l_msg);

            fclose(f);

            result = RESULT_SUCCESS;

            // Generates response message with file content
            response = message_file_receive_response( result, strlen(buffer), buffer, &response_len );

            // Sends a File Receive response message
            if ( !process_outgoing_message( proc_data->connection, response, response_len ) ) {
    
              LOGGER(__FUNCTION__, "File Receive message could not be sent.");
            }
            
            // Deletes generated temporary files
            remove(gzip_output);
            remove(b64_output);

            // Frees response
            free(response);

          } else {

            sprintf(l_msg, "Error opening file (%s)", b64_output);
            LOGGER(__FUNCTION__, l_msg);

            result = RESULT_FILE_READ_ERROR;
          }

          // Frees buffer
          free(buffer);
          
        } else {

          sprintf(l_msg, "Error encoding file (%s)", gzip_output);
          LOGGER(__FUNCTION__, l_msg);

          result = RESULT_FILE_ENCODE_ERROR;
        }

      } else {

        sprintf(l_msg, "Error packing file (%s)", filename);
        LOGGER(__FUNCTION__, l_msg);

        result = RESULT_FILE_COMPRESS_ERROR;
      }

    }

  }

END_PROCESS_FILE_RECEIVE:

  // Cleanup
  if (request != NULL) {
    free(request);
  }
  if (filename != NULL) {
    free(filename);
  }

  if (result == RESULT_SUCCESS) {

    return;
  }
  else {

    // Generates response
    response = message_file_receive_response( result, 0, NULL, &response_len );

    // Sends a File Receive response message
    if ( !process_outgoing_message( proc_data->connection, response, response_len ) ) {
    
      LOGGER(__FUNCTION__, "File Receive response message could not be sent.");
    }

    // Libera la respuesta
    free(response);
  }

  return;
}
Пример #3
0
/**
 * Processes a File Delete message from the client, 
 * performs and finalizes the operation
 *
 * @param proc_data_arg           data structure with connection parameters 
 *                                and received message
 */
void process_file_delete( PROCESS_DATA_T * proc_data ) {

  const int NOT_FOUND = -1;

  char * filename   = NULL;
  char l_msg[4096];

  char * request    = NULL;
  char * response   = NULL;

  unsigned long response_len  = 0;
  int search_filename_res     = 0;

  int result = RESULT_UNDEFINED;
  const int param_len = (proc_data->received_msg_len - HEADER_LEN - strlen(PARAM_FILENAME));

  // Copies the request to a safe buffer for searching
  request = (char*)malloc(sizeof(char) * proc_data->received_msg_len + 1);
  memcpy(request, proc_data->received_message, proc_data->received_msg_len);
  request[proc_data->received_msg_len] = '\0';

  // Gets name of file to delete
  search_filename_res = STR_SEARCH( proc_data->received_message, PARAM_FILENAME, HEADER_LEN);
  if ( search_filename_res == NOT_FOUND ) {

    result = RESULT_INVALID_REQUEST;
    goto END_PROCESS_FILE_DELETE;
  }

  filename = (char*)malloc(sizeof(char) * proc_data->received_msg_len);
  memcpy( filename, &(request[ ( search_filename_res + strlen(PARAM_FILENAME) ) ]), param_len );
  filename[param_len] = '\0';
  
  sprintf(l_msg, "A request has been received to delete the file: %s", filename);
  LOGGER(__FUNCTION__, l_msg);
  
  if (file_exists(filename) == TRUE) {
  
    if (file_delete(filename) == TRUE) {
    
      result = RESULT_SUCCESS;
    }
    else {
      result = RESULT_FILE_DELETE_ERROR;
    }    
  }
  else {
    result = RESULT_FILE_NOT_FOUND;
  }
    
  // Generates response message
  response = message_file_delete_response( result, &response_len );  

  // Sends response message
  if ( !process_outgoing_message( proc_data->connection, response, response_len ) ) {
    
    LOGGER(__FUNCTION__, "File Receive response message could not be sent.");
  }

END_PROCESS_FILE_DELETE:

  // Cleanup
  if (request != NULL) {
    free(request);
  }
  if (response != NULL) {
    free(response);
  }
  if (filename != NULL) {
    free(filename);
  }

  return;
}
Пример #4
0
	void stateOperatorControl() {
		// DRIVING
		move = stick->GetRawAxis(1) * -1.0;
		rotate = stick->GetRawAxis(4) * -1.0;

		// Deadband
		if (fabs(move) < 0.1) {
			move = 0.0;
		}
		if (fabs(rotate) < 0.15) {
			rotate = 0.0;
		}
		drive->ArcadeDrive(move, rotate, false);

		// Joystick Buttons
		bool button4 = stick->GetRawButton(4);
		bool button1 = stick->GetRawButton(1);
		bool button5 = stick->GetRawButton(5);
		bool button6 = stick->GetRawButton(6);
		bool button3 = stick->GetRawButton(3);

		// Manual Gatherer
		if (stick->GetRawAxis(2) != 0) {
			gatherSpeed = stick->GetRawAxis(2);
			LOGGER(DEBUG) << "[stateOperatorControl] Gather Angle:" << gatherPIDSource.PIDGet();

		}
		else if (stick->GetRawAxis(3) != 0) {
			gatherSpeed = stick->GetRawAxis(3) * -1;
			LOGGER(DEBUG) << "[stateOperatorControl] Gather Angle:" << gatherPIDSource.PIDGet();
		}
		else {
			gatherSpeed = 0.0;
		}
		gatherer->Set(gatherSpeed);

		// Launch Angle
		double launcherAngle = launchPIDSource.PIDGet();
		if (button5 && !button6  && (launcherAngle < kLaunchMaxAngle)) {
			elevator->Set(-0.5); // Up
			LOGGER(DEBUG) << "[stateOperatorControl] Launcher Angle:" << launcherAngle;
		} else if (button6 && !button5 && (launcherAngle > kLaunchMinAngle)) {
			LOGGER(DEBUG) << "[stateOperatorControl] Launcher Angle:" << launcherAngle;
			elevator->Set(0.5); // Down
		} else {
			elevator->Set(0.0);
		}

		// Auto-Gather
		if (button3 && !lastButton3) {
			wheelsGathererIn = !wheelsGathererIn;
			gatherController->SetSetpoint(kGatherAngle);
			gatherController->Enable();
		}
		if (wheelsGathererIn) {
			gathererWheels->Set(1.0);
			gatherer->Set(gatherPIDOutput.correction);
			LOGGER(DEBUG) << "[stateOperatorControl] Gather Correction:" << gatherPIDOutput.correction
					      << " Gather Angle: " << gatherPIDSource.PIDGet();
		} else {
			gathererWheels->Set(0.0);
			gatherController->Disable();
		}

		if (button4 && !lastButton4) {
			stateTimer   = 0;
			robotState   = kCentering;
			shootingHigh = true;
		}
		if (button1 && !lastButton1) {
			stateTimer   = 0;
			robotState   = kLaunching;
			shootingHigh = true;
		}
		lastButton4 = button4;
		lastButton1 = button1;
		lastButton3 = button3;
	}
Пример #5
0
	MCFG_CPU_PERIODIC_INT_DRIVER(midcoin24cdjuke_state, irq0_line_hold, 500)

	MCFG_DEFAULT_LAYOUT(layout_24cdjuke)

	MCFG_DEVICE_ADD("ic11", I8255A, 0)
	MCFG_I8255_IN_PORTA_CB(IOPORT("MD1"))
	MCFG_I8255_IN_PORTB_CB(IOPORT("MD2"))
	MCFG_I8255_IN_PORTC_CB(IOPORT("MD3"))

	MCFG_DEVICE_ADD("ic25", I8255A, 0)
	MCFG_I8255_IN_PORTB_CB(IOPORT("PB"))
	MCFG_I8255_IN_PORTC_CB(READ8(midcoin24cdjuke_state, kb_row_r))
	MCFG_I8255_OUT_PORTC_CB(WRITE8(midcoin24cdjuke_state, kb_col_w))

	MCFG_DEVICE_ADD("ic31", I8255A, 0)
	MCFG_I8255_OUT_PORTB_CB(LOGGER("PPI8255 - unmapped write port B", 0))
	MCFG_I8255_IN_PORTC_CB(IOPORT("MD4"))
MACHINE_CONFIG_END


ROM_START( 24cdjuke )
	ROM_REGION( 0x4000, "maincpu", 0 )
	ROM_LOAD( "1.ic5", 0x0000, 0x4000,  CRC(df2419ad) SHA1(dd9dd85011d46581dccabcfdb5959a8b018df937)  )

	ROM_REGION16_LE( 0x200, "charset", 0 )
	ROM_LOAD16_BYTE( "dm74ls471n.ic20", 0x000, 0x100, CRC(d05765e6) SHA1(119ec6ca1a4afa0ea6ab1020ba2a8b02fd434e3f) )
	ROM_LOAD16_BYTE( "dm74ls471n.ic21", 0x001, 0x100, CRC(e12d5a04) SHA1(be52ee4e4a5ea225fce39c759645a7cf49cea370) )

	// MAB8441T-T042 internal ROM?
	ROM_REGION( 0x80000, "misc", 0 )
	ROM_LOAD( "m1-7611a-5.ic27", 0x000, 0x100, CRC(29b068e8) SHA1(477e2445c58b7d14c56a3ad4050eb22474d56005) )
Пример #6
0
/**
 * Checks if a socket is available for reading and/or writing
 *
 * @param timeout               timeout, or 0 for blocking
 * @param socket                socket to check state on
 * @param operation_type        operation type: S_READ, S_WRITE, S_RW
 *
 * @return                      type of operation the socket is available for,
 *                              can be S_READ, S_WRITE, S_RW, 
 *                              or -1 if an error occurred
 */
int socket_select(int timeout, SOCKET_T * select_socket, int operation_type) {

  fd_set* readfds = NULL;
  fd_set* writefds = NULL;
  struct timeval tval_timeout;
  char buffer[1024];
  int retval = 0;
  int res;

  // Validates parameters
  if ( ( select_socket == NULL ) || ( (operation_type != S_READ) && (operation_type != S_WRITE) && (operation_type != S_RW) ) ) {

    sprintf(buffer, "socket_select fail: invalid parameters");
    LOGGER(__FUNCTION__, buffer);
    return -1;

  }

  // Configures timeout
  tval_timeout.tv_sec = timeout;
  tval_timeout.tv_usec = 0;

  // Allocates memory for sets
  readfds = (fd_set*)malloc(sizeof(fd_set));
  writefds = (fd_set*)malloc(sizeof(fd_set));

  // Initializes sets
  FD_ZERO(readfds);
  FD_ZERO(writefds);

  // Locks the socket's mutex
  MUTEX_LOCK(select_socket->mutex);

  // If has to check for reading
  if (operation_type & S_READ) {
    
    // Adds socket to the read set
    FD_SET( select_socket->handle, readfds );
  }

  // If has to check for writing
  if (operation_type & S_WRITE) {

    // Adds socket to the write set
    FD_SET(select_socket->handle, writefds);
  }

  // Calls select function
  res = select( (select_socket->handle)+1, readfds, writefds, NULL, &tval_timeout );
  if (res == -1) {

    sprintf(buffer, "select failed with error: %d\n", errno);
    LOGGER(__FUNCTION__, buffer);
    
    retval = -1;
    goto end_select;
  }

  // If socket is on the read result set
  // adds to return value
  if ( FD_ISSET(select_socket->handle, readfds ) ) {
    retval += S_READ;
  }

  // If socket is on the write result set
  // adds to return value
  if ( FD_ISSET(select_socket->handle, writefds ) ) {
    retval += S_WRITE;
  }

end_select:

  // Unlocks the socket's mutex
  MUTEX_UNLOCK(select_socket->mutex);

  // Frees allocated memory
  free(readfds);
  free(writefds);

  return retval;

}
Пример #7
0
/**
 * Creates a new socket and connects to server if it is a client,
 * or does the binding and starts listening on a port if it is server.
 * Returns a reference to the newly created connection.
 *
 * @param side                  0 if it is a client, 1 if it is a server
 * @param addr                  address to connect for clients or to listen on
 *                              for server, if NULL for server listens on INADDR_ANY
 * @param port                  port to connect or to listen on
 * @param max_connections       max number of connections that the server can
 *                              accept. does not apply for clients.
 * @param nonblocking           TRUE if nonblocking is desired, otherwise FALSE
 * 
 * @return                      pointer to the newly created socket, on error
 *                              returns NULL
 */
SOCKET_T* socket_create(int side, char* addr, int port, int max_connections, int nonblocking) {

  SOCKET_T* new_socket = NULL;
  struct sockaddr_in service;
  
  char buffer[_BUFFER_SIZE_S];
  int res;

  int socket_handle;

  // Validates paramter
  if (side < 0 || side > 1) {
    sprintf(buffer, "socket_create fail: parametros no validos");
    return NULL;
  }

  // Configurates the service
  service.sin_family = AF_INET;
  service.sin_port = htons((unsigned short)port);
  if (addr == NULL) {
    service.sin_addr.s_addr = htonl(INADDR_ANY);
  } else {
    service.sin_addr.s_addr = inet_addr(addr);
  }

  //
  // Creates socket and does the binding and listening
  // operations for server or connect for clients
  //

  // Creates socket
  if (nonblocking == TRUE) {
    socket_handle = socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0);
  } else {
    socket_handle = socket(AF_INET, SOCK_STREAM, 0);
  }
    
  if (socket_handle == -1) {
      
    sprintf(buffer, "socket failed with error: %d\n", errno);
    LOGGER(__FUNCTION__, buffer);
    return NULL;
  }  

  // If it is server
  if (side == 1) {

    // Binds the socket to the port
    res = bind (socket_handle, (struct sockaddr *) &service, sizeof (service));
    if (res == -1)
    {
      sprintf(buffer, "socket failed with error: %d\n", errno);
      LOGGER(__FUNCTION__, buffer);

      shutdown(socket_handle, SHUT_RDWR);
      return NULL;
    }

    // Starts listening on the port
    res = listen(socket_handle, max_connections);
    if (res == -1) {

      sprintf(buffer, "listen fallo con el error: %d\n", errno);
      LOGGER(__FUNCTION__, buffer);
      
      shutdown(socket_handle, SHUT_RDWR);      
      return NULL;
    }
  
  }
  // If it is client
  else {

    // Connects to the server
    res = connect(socket_handle, (struct sockaddr *) &service, sizeof (service));
    if ( res == -1) {

      // Unless connection is in progress
      if ( errno != EINPROGRESS ) {

        sprintf(buffer, "connect failed with error: %d\n", errno);
        LOGGER(__FUNCTION__, buffer);
      
        shutdown(socket_handle, SHUT_RDWR);      
        return NULL;  
      } 

    }
  
  }

  // Configures timeouts for read/write
  {
    struct timeval timeout;      
    timeout.tv_sec = RW_TIMEOUT;
    timeout.tv_usec = 0;

    res = setsockopt(socket_handle, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(timeout));
    if (res == -1) {
      
      sprintf(buffer, "setsockopt failed with error: %d\n", errno);
      LOGGER(__FUNCTION__, buffer);
    }

    res = setsockopt(socket_handle, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(timeout));
    if (res == -1) {
      
      sprintf(buffer, "setsockopt failed with error: %d\n", errno);
      LOGGER(__FUNCTION__, buffer);
    }
  }

  // Allocates space for the socket structure
  new_socket = malloc(sizeof( SOCKET_T ));
  memset(new_socket, 0x00, sizeof( SOCKET_T ));

  // Creates mutex for thread-safe socket
  new_socket->mutex = (MUTEX_T*)malloc(sizeof(struct _mutex_t) );
  memset(new_socket->mutex, 0x00, sizeof(struct _mutex_t));
  MUTEX_CREATE(&new_socket->mutex);
  
  // Copies handle to the socket structure
  new_socket->handle = socket_handle;
    
  return new_socket;
  
}
Пример #8
0
ibis::table* ibis::jRange::select(const char *sstr) const {
    if (nrows < 0) {
        int64_t ierr = count();
        if (ierr < 0) {
            LOGGER(ibis::gVerbose > 0)
                << "Warning -- jRange::count failed with error code"
                << ierr;
            return 0;
        }
    }
    if (sstr == 0 || *sstr == 0) { // default
        std::string tn = ibis::util::shortName(desc_.c_str());
        return new ibis::tabula(tn.c_str(), desc_.c_str(), nrows);
    }

    ibis::selectClause sel(sstr);
    uint32_t features=0; // 1: arithmetic operation, 2: aggregation
    // use a barrel to collect all unique names
    ibis::math::barrel brl;
    for (uint32_t j = 0; j < sel.aggSize(); ++ j) {
        const ibis::math::term* t = sel.aggExpr(j);
        brl.recordVariable(t);
        if (t->termType() != ibis::math::VARIABLE &&
            t->termType() != ibis::math::NUMBER &&
            t->termType() != ibis::math::STRING) {
            features |= 1; // arithmetic operation
        }
        if (sel.getAggregator(j) != ibis::selectClause::NIL_AGGR) {
            features |= 2; // aggregation
        }
    }
    // convert the barrel into a stringArray for processing
    ibis::table::stringArray sl;
    sl.reserve(brl.size());
    for (unsigned j = 0; j < brl.size(); ++ j) {
        const char* str = brl.name(j);
        if (*str != 0) {
            if (str[0] != '_' || str[1] != '_')
                sl.push_back(str);
        }
    }

    std::unique_ptr<ibis::table> res1(select(sl));
    if (res1.get() == 0 || res1->nRows() == 0 || res1->nColumns() == 0 ||
        features == 0)
        return res1.release();

    if (ibis::gVerbose > 2) {
        ibis::util::logger lg;
        lg() << "jRange::select(" << sstr << ", " << desc_
             << ") produced the first intermediate table:\n";
        res1->describe(lg());
    }

    if ((features & 1) != 0) { // arithmetic computations
        res1.reset(static_cast<const ibis::bord*>(res1.get())->evaluateTerms
                   (sel, desc_.c_str()));
        if (res1.get() != 0) {
            if (ibis::gVerbose > 2) {
                ibis::util::logger lg;
                lg() << "jRange::select(" << sel << ", " << desc_
                     << ") produced the second intermediate table:\n";
                res1->describe(lg());
            }
        }
        else {
            LOGGER(ibis::gVerbose > 0)
                << "Warning -- jRange::select(" << sel
                << ") failed to evaluate the arithmetic expressions";
            return 0;
        }
    }

    if ((features & 2) != 0) { // aggregation operations
        res1.reset(static_cast<const ibis::bord*>(res1.get())->groupby(sel));
        if (res1.get() != 0) {
            if (ibis::gVerbose > 2) {
                ibis::util::logger lg;
                lg() << "jRange::select(" << *sel_ << ", " << desc_
                     << ") produced the third intermediate table:\n";
                res1->describe(lg());
            }
        }
        else {
            LOGGER(ibis::gVerbose > 0)
                << "Warning -- jRange::select(" << *sel_
                << ") failed to evaluate the aggregations";
        }
    }
    return res1.release();
} // ibis::jRange::select
Пример #9
0
ibis::table*
ibis::jRange::select(const ibis::table::stringArray& colnames) const {
    ibis::table *res = 0;
    if (nrows < 0) {
        int64_t ierr = count();
        if (ierr < 0) {
            LOGGER(ibis::gVerbose > 0)
                << "Warning -- jRange::count failed with error code"
                << ierr;
            return res;
        }
    }
    if (valr_ == 0 || orderr_ == 0 || vals_ == 0 || orders_ == 0 ||
        orderr_->size() != maskr_.cnt() || orders_->size() != masks_.cnt()) {
        LOGGER(ibis::gVerbose > 0)
            << "Warning -- jRange::select failed to evaluate the join";
        return res;
    }
    if (colnames.empty() || nrows == 0) {
        std::string nm = ibis::util::shortName(desc_);
        res = new ibis::tabula(nm.c_str(), desc_.c_str(), nrows);
        return res;
    }

    const uint32_t ncols = colnames.size();
    std::string evt;
    evt = "select ";
    evt += colnames[0];
    for (uint32_t j = 1; j < ncols; ++ j) {
        evt += ", ";
        evt += colnames[j];
    }
    if ((desc_[0] != 'F' && desc_[0] != 'f') ||
        (desc_[1] != 'R' && desc_[1] != 'r') ||
        (desc_[2] != 'O' && desc_[2] != 'o') ||
        (desc_[3] != 'M' && desc_[3] != 'm'))
        evt += " for ";
    else
        evt += ' ';
    evt += desc_;
    ibis::util::timer mytimer(evt.c_str());
    std::map<const char*, uint32_t, ibis::lessi> namesToPos;
    std::vector<uint32_t> ipToPos(colnames.size());
    std::vector<const ibis::column*> ircol, iscol;
    std::vector<const ibis::dictionary*> cats(colnames.size(), 0);
    // identify the names from the two data partitions
    for (uint32_t j = 0; j < ncols; ++ j) {
        ipToPos[j] = ncols+1;
        const char* cn = colnames[j];
        std::string tname;
        while (*cn != 0 && *cn != '.') {
            tname += *cn;
            ++ cn;
        }
        if (*cn == '.') {
            ++ cn;
        }
        else { // did not find '.'
            tname.erase();
            cn = colnames[j];
        }
        int match = -1; // 0 ==> partr_, 1 ==> parts_
        if (! tname.empty()) {
            match = frm_->position(tname.c_str());
            if (match >= static_cast<long>(frm_->size())) {
                if (stricmp(tname.c_str(), partr_.name()) == 0) {
                    match = 0;
                }
                else if (stricmp(tname.c_str(), parts_.name()) == 0) {
                    match = 1;
                }
            }
        }

        if (match == 0) {
            const ibis::column *col = partr_.getColumn(cn);
            if (col != 0) {
                namesToPos[colnames[j]] = j;
                ipToPos[j] = ircol.size();
                ircol.push_back(col);
                if (col->type() == ibis::CATEGORY) {
                    const ibis::category *cat =
                        static_cast<const ibis::category*>(col);
                    cats[j] = cat->getDictionary();
                }
                else if (col->type() == ibis::UINT) {
                    const ibis::bord::column *bc =
                        dynamic_cast<const ibis::bord::column*>(col);
                    if (bc != 0) {
                        cats[j] = bc->getDictionary();
                    }
                }
            }
            else {
                LOGGER(ibis::gVerbose > 0)
                    << "Warning -- " << evt << " can not find column named \""
                    << colnames[j] << "\" in data partition \"" << partr_.name()
                    << "\"";
                return res;
            }
        }
        else if (match == 1) {
            const ibis::column *col = parts_.getColumn(cn);
            if (col != 0) {
                namesToPos[colnames[j]] = j;
                ipToPos[j] = ncols - iscol.size();
                iscol.push_back(col);
                if (col->type() == ibis::CATEGORY) {
                    const ibis::category *cat =
                        static_cast<const ibis::category*>(col);
                    cats[j] = cat->getDictionary();
                }
                else if (col->type() == ibis::UINT) {
                    const ibis::bord::column *bc =
                        dynamic_cast<const ibis::bord::column*>(col);
                    if (bc != 0) {
                        cats[j] = bc->getDictionary();
                    }
                }
            }
            else {
                LOGGER(ibis::gVerbose > 0)
                    << "Warning -- " << evt << " can not find column named \""
                    << colnames[j] << "\" in data partition \""
                    << parts_.name() << "\"";
                return res;
            }
        }
        else { // not prefixed with a data partition name
            cn = colnames[j];
            const ibis::column* col = partr_.getColumn(cn);
            if (col != 0) {
                ipToPos[j] = ircol.size();
                ircol.push_back(col);
                if (col->type() == ibis::CATEGORY) {
                    const ibis::category *cat =
                        static_cast<const ibis::category*>(col);
                    cats[j] = cat->getDictionary();
                }
                else if (col->type() == ibis::UINT) {
                    const ibis::bord::column *bc =
                        dynamic_cast<const ibis::bord::column*>(col);
                    if (bc != 0) {
                        cats[j] = bc->getDictionary();
                    }
                }
                LOGGER(ibis::gVerbose > 3)
                    << evt << " encountered a column name ("
                    << colnames[j] << ") that does not start with a data "
                    "partition name, assume it is for \"" << partr_.name()
                    << "\"";
            }
            else {
                col = parts_.getColumn(cn);
                if (col != 0) {
                    ipToPos[j] = ncols - iscol.size();
                    iscol.push_back(col);
                    if (col->type() == ibis::CATEGORY) {
                        const ibis::category *cat =
                            static_cast<const ibis::category*>(col);
                        cats[j] = cat->getDictionary();
                    }
                    else if (col->type() == ibis::UINT) {
                        const ibis::bord::column *bc =
                            dynamic_cast<const ibis::bord::column*>(col);
                        if (bc != 0) {
                            cats[j] = bc->getDictionary();
                        }
                    }
                    LOGGER(ibis::gVerbose > 1)
                        << evt << " encountered a column name (" << colnames[j]
                        << ") that does not start with a data partition name, "
                        "assume it is for \"" << parts_.name() << "\"";
                }
                else {
                    LOGGER(ibis::gVerbose > 0)
                        << "Warning -- " << evt << " encountered a name ("
                        << colnames[j] << ") that does not start with a data "
                        "partition name";
                    return res;
                }
            }
        }
    } // for (uint32_t j = 0; j < ncols;

    LOGGER(ibis::gVerbose > 3)
        << evt << " -- found " << ircol.size()
        << " column" << (ircol.size() > 1 ? "s" : "") << " from "
        << partr_.name() << " and " << iscol.size() << " column"
        << (iscol.size() > 1 ? "s" : "") << " from " << parts_.name();

    // change Pos values for columns in S to have offset ircol.size()
    for (uint32_t j = 0; j < ncols; ++j) {
        if (ipToPos[j] <= ncols && ipToPos[j] >= ircol.size())
            ipToPos[j] = (ncols - ipToPos[j]) + ircol.size();
    }
    ibis::table::typeArray   rtypes(ircol.size(), ibis::UNKNOWN_TYPE);
    ibis::table::bufferArray rbuff(ircol.size(), 0);
    IBIS_BLOCK_GUARD(ibis::table::freeBuffers, ibis::util::ref(rbuff),
                     ibis::util::ref(rtypes));
    ibis::table::typeArray   stypes(iscol.size(), ibis::UNKNOWN_TYPE);
    ibis::table::bufferArray sbuff(iscol.size(), 0);
    IBIS_BLOCK_GUARD(ibis::table::freeBuffers, ibis::util::ref(sbuff),
                     ibis::util::ref(stypes));
    bool sane = true;

    // retrieve values from r_
    for (uint32_t j = 0; sane && j < ircol.size(); ++ j) {
        rtypes[j] = ircol[j]->type();
        switch (ircol[j]->type()) {
        case ibis::BYTE:
            rbuff[j] = ircol[j]->selectBytes(maskr_);
            if (rbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<signed char>*>(rbuff[j]),
                     *orderr_);
            else
                sane = false;
            break;
        case ibis::UBYTE:
            rbuff[j] = ircol[j]->selectUBytes(maskr_);
            if (rbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<unsigned char>*>(rbuff[j]),
                     *orderr_);
            else
                sane = false;
            break;
        case ibis::SHORT:
            rbuff[j] = ircol[j]->selectShorts(maskr_);
            if (rbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<int16_t>*>(rbuff[j]),
                     *orderr_);
            else
                sane = false;
            break;
        case ibis::USHORT:
            rbuff[j] = ircol[j]->selectUShorts(maskr_);
            if (rbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<uint16_t>*>(rbuff[j]),
                     *orderr_);
            else
                sane = false;
            break;
        case ibis::INT:
            rbuff[j] = ircol[j]->selectInts(maskr_);
            if (rbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<int32_t>*>(rbuff[j]),
                     *orderr_);
            else
                sane = false;
            break;
        case ibis::UINT:
            rbuff[j] = ircol[j]->selectUInts(maskr_);
            if (rbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<uint32_t>*>(rbuff[j]),
                     *orderr_);
            else
                sane = false;
            break;
        case ibis::LONG:
            rbuff[j] = ircol[j]->selectLongs(maskr_);
            if (rbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<int64_t>*>(rbuff[j]),
                     *orderr_);
            else
                sane = false;
            break;
        case ibis::ULONG:
            rbuff[j] = ircol[j]->selectULongs(maskr_);
            if (rbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<uint64_t>*>(rbuff[j]),
                     *orderr_);
            else
                sane = false;
            break;
        case ibis::FLOAT:
            rbuff[j] = ircol[j]->selectFloats(maskr_);
            if (rbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<float>*>(rbuff[j]),
                     *orderr_);
            else
                sane = false;
            break;
        case ibis::DOUBLE:
            rbuff[j] = ircol[j]->selectDoubles(maskr_);
            if (rbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<double>*>(rbuff[j]),
                     *orderr_);
            else
                sane = false;
            break;
        case ibis::TEXT:
        case ibis::CATEGORY:
            rbuff[j] = ircol[j]->selectStrings(maskr_);
            if (rbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<std::vector<std::string>*>(rbuff[j]),
                     *orderr_);
            else
                sane = false;
            break;
        default:
            sane = false;
            rbuff[j] = 0;
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::select does not support column "
                "type " << ibis::TYPESTRING[(int)ircol[j]->type()]
                << " (name = " << partr_.name() << "." << ircol[j]->name()
                << ")";
            break;
        }
    }
    if (! sane) {
        return res;
    }

    // retrieve values from parts_
    for (uint32_t j = 0; sane && j < iscol.size(); ++ j) {
        stypes[j] = iscol[j]->type();
        switch (iscol[j]->type()) {
        case ibis::BYTE:
            sbuff[j] = iscol[j]->selectBytes(masks_);
            if (sbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<signed char>*>(sbuff[j]),
                     *orders_);
            else
                sane = false;
            break;
        case ibis::UBYTE:
            sbuff[j] = iscol[j]->selectUBytes(masks_);
            if (sbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<unsigned char>*>(sbuff[j]),
                     *orders_);
            else
                sane = false;
            break;
        case ibis::SHORT:
            sbuff[j] = iscol[j]->selectShorts(masks_);
            if (sbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<int16_t>*>(sbuff[j]),
                     *orders_);
            else
                sane = false;
            break;
        case ibis::USHORT:
            sbuff[j] = iscol[j]->selectUShorts(masks_);
            if (sbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<uint16_t>*>(sbuff[j]),
                     *orders_);
            else
                sane = false;
            break;
        case ibis::INT:
            sbuff[j] = iscol[j]->selectInts(masks_);
            if (sbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<int32_t>*>(sbuff[j]),
                     *orders_);
            else
                sane = false;
            break;
        case ibis::UINT:
            sbuff[j] = iscol[j]->selectUInts(masks_);
            if (sbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<uint32_t>*>(sbuff[j]),
                     *orders_);
            else
                sane = false;
            break;
        case ibis::LONG:
            sbuff[j] = iscol[j]->selectLongs(masks_);
            if (sbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<int64_t>*>(sbuff[j]),
                     *orders_);
            else
                sane = false;
            break;
        case ibis::ULONG:
            sbuff[j] = iscol[j]->selectULongs(masks_);
            if (sbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<uint64_t>*>(sbuff[j]),
                     *orders_);
            else
                sane = false;
            break;
        case ibis::FLOAT:
            sbuff[j] = iscol[j]->selectFloats(masks_);
            if (sbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<float>*>(sbuff[j]),
                     *orders_);
            else
                sane = false;
            break;
        case ibis::DOUBLE:
            sbuff[j] = iscol[j]->selectDoubles(masks_);
            if (sbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<array_t<double>*>(sbuff[j]),
                     *orders_);
            else
                sane = false;
            break;
        case ibis::TEXT:
        case ibis::CATEGORY:
            sbuff[j] = iscol[j]->selectStrings(masks_);
            if (sbuff[j] != 0)
                ibis::util::reorder
                    (*static_cast<std::vector<std::string>*>(sbuff[j]),
                     *orders_);
            else
                sane = false;
            break;
        default:
            sane = false;
            sbuff[j] = 0;
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::select does not support column "
                "type " << ibis::TYPESTRING[(int)iscol[j]->type()]
                << " (name = " << parts_.name() << "." << iscol[j]->name()
                << ")";
            break;
        }
    }
    if (! sane) {
        return res;
    }

    /// fill the in-memory buffer
    switch (colr_.type()) {
    case ibis::BYTE:
        res = fillResult
            (nrows, delta1_, delta2_, evt,
             *static_cast<array_t<signed char>*>(valr_), rtypes, rbuff,
             *static_cast<array_t<signed char>*>(vals_), stypes, sbuff,
             colnames, ipToPos);
        break;
    case ibis::UBYTE:
        res = fillResult
            (nrows, delta1_, delta2_, evt,
             *static_cast<array_t<unsigned char>*>(valr_), rtypes, rbuff,
             *static_cast<array_t<unsigned char>*>(vals_), stypes, sbuff,
             colnames, ipToPos);
        break;
    case ibis::SHORT:
        res = fillResult
            (nrows, delta1_, delta2_, evt,
             *static_cast<array_t<int16_t>*>(valr_), rtypes, rbuff,
             *static_cast<array_t<int16_t>*>(vals_), stypes, sbuff,
             colnames, ipToPos);
        break;
    case ibis::USHORT:
        res = fillResult
            (nrows, delta1_, delta2_, evt,
             *static_cast<array_t<uint16_t>*>(valr_), rtypes, rbuff,
             *static_cast<array_t<uint16_t>*>(vals_), stypes, sbuff,
             colnames, ipToPos);
        break;
    case ibis::INT:
        res = fillResult
            (nrows, delta1_, delta2_, evt,
             *static_cast<array_t<int32_t>*>(valr_), rtypes, rbuff,
             *static_cast<array_t<int32_t>*>(vals_), stypes, sbuff,
             colnames, ipToPos);
        break;
    case ibis::UINT:
        res = fillResult
            (nrows, delta1_, delta2_, evt,
             *static_cast<array_t<uint32_t>*>(valr_), rtypes, rbuff,
             *static_cast<array_t<uint32_t>*>(vals_), stypes, sbuff,
             colnames, ipToPos);
        break;
    case ibis::LONG:
        res = fillResult
            (nrows, delta1_, delta2_, evt,
             *static_cast<array_t<int64_t>*>(valr_), rtypes, rbuff,
             *static_cast<array_t<int64_t>*>(vals_), stypes, sbuff,
             colnames, ipToPos);
        break;
    case ibis::ULONG:
        res = fillResult
            (nrows, delta1_, delta2_, evt,
             *static_cast<array_t<uint64_t>*>(valr_), rtypes, rbuff,
             *static_cast<array_t<uint64_t>*>(vals_), stypes, sbuff,
             colnames, ipToPos);
        break;
    case ibis::FLOAT:
        res = fillResult
            (nrows, delta1_, delta2_, evt,
             *static_cast<array_t<float>*>(valr_), rtypes, rbuff,
             *static_cast<array_t<float>*>(vals_), stypes, sbuff,
             colnames, ipToPos);
        break;
    case ibis::DOUBLE:
        res = fillResult
            (nrows, delta1_, delta2_, evt,
             *static_cast<array_t<double>*>(valr_), rtypes, rbuff,
             *static_cast<array_t<double>*>(vals_), stypes, sbuff,
             colnames, ipToPos);
        break;
    default:
        LOGGER(ibis::gVerbose > 0)
            << "Warning -- " << evt << " can not handle join column of type "
            << ibis::TYPESTRING[(int)colr_.type()];
    }

    for (unsigned j = 0; j < cats.size(); ++ j) {
        if (cats[j] != 0) {
            ibis::bord::column *bc = dynamic_cast<ibis::bord::column*>
                (static_cast<ibis::bord*>(res)->getColumn(j));
            if (bc != 0)
                bc->setDictionary(cats[j]);
        }
    }
    return res;
} // ibis::jRange::select
Пример #10
0
int64_t ibis::jRange::count() const {
    if (nrows >= 0) return nrows; // already have done this
    if (maskr_.cnt() == 0 || masks_.cnt() == 0) {
        return 0;
    }

    std::string mesg;
    mesg = "jRange::count(";
    mesg += desc_;
    mesg += ")";
    ibis::util::timer tm(mesg.c_str(), 1);
    // allocate space for ordering arrays
    orderr_ = new array_t<uint32_t>;
    orders_ = new array_t<uint32_t>;

    // Retrieve and sort the values
    switch (colr_.type()) {
    default:
        LOGGER(ibis::gVerbose > 1)
            << "Warning -- jRange[" << desc_
            << "] cann't handle join column of type " << colr_.type();
        return -2;
    case ibis::BYTE: {
        valr_ = colr_.selectBytes(maskr_);
        if (valr_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << colr_.name() << "->selectBytes("
                << maskr_.cnt() << ") failed";
            return -3;
        }
        vals_ = cols_.selectBytes(masks_);
        if (vals_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << cols_.name() << "->selectBytes("
                << masks_.cnt() << ") failed";
            return -4;
        }
        nrows = ibis::util::sortMerge
            (*static_cast<array_t<signed char>*>(valr_), *orderr_,
             *static_cast<array_t<signed char>*>(vals_), *orders_,
             delta1_, delta2_);
        break;}
    case ibis::UBYTE: {
        valr_ = colr_.selectUBytes(maskr_);
        if (valr_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << colr_.name() << "->selectUBytes("
                << maskr_.cnt() << ") failed";
            return -3;
        }
        vals_ = cols_.selectUBytes(masks_);
        if (vals_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << cols_.name() << "->selectUBytes("
                << masks_.cnt() << ") failed";
            return -4;
        }
        nrows = ibis::util::sortMerge
            (*static_cast<array_t<unsigned char>*>(valr_),
             *orderr_,
             *static_cast<array_t<unsigned char>*>(vals_),
             *orders_, delta1_, delta2_);
        break;}
    case ibis::SHORT: {
        valr_ = colr_.selectShorts(maskr_);
        if (valr_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << colr_.name() << "->selectShorts("
                << maskr_.cnt() << ") failed";
            return -3;
        }
        vals_ = cols_.selectShorts(masks_);
        if (vals_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << cols_.name() << "->selectShorts("
                << masks_.cnt() << ") failed";
            return -4;
        }
        nrows = ibis::util::sortMerge
            (*static_cast<array_t<int16_t>*>(valr_), *orderr_,
             *static_cast<array_t<int16_t>*>(vals_), *orders_,
             delta1_, delta2_);
        break;}
    case ibis::USHORT: {
        valr_ = colr_.selectUShorts(maskr_);
        if (valr_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << colr_.name() << "->selectUShorts("
                << maskr_.cnt() << ") failed";
            return -3;
        }
        vals_ = cols_.selectUShorts(masks_);
        if (vals_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << cols_.name() << "->selectUShorts("
                << masks_.cnt() << ") failed";
            return -4;
        }
        nrows = ibis::util::sortMerge
            (*static_cast<array_t<uint16_t>*>(valr_), *orderr_,
             *static_cast<array_t<uint16_t>*>(vals_), *orders_,
             delta1_, delta2_);
        break;}
    case ibis::INT: {
        valr_ = colr_.selectInts(maskr_);
        if (valr_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << colr_.name() << "->selectInts("
                << maskr_.cnt() << ") failed";
            return -3;
        }
        vals_ = cols_.selectInts(masks_);
        if (vals_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << cols_.name() << "->selectInts("
                << masks_.cnt() << ") failed";
            return -4;
        }
        nrows = ibis::util::sortMerge
            (*static_cast<array_t<int32_t>*>(valr_), *orderr_,
             *static_cast<array_t<int32_t>*>(vals_), *orders_,
             delta1_, delta2_);
        break;}
    case ibis::UINT: {
        valr_ = colr_.selectUInts(maskr_);
        if (valr_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << colr_.name() << "->selectUInts("
                << maskr_.cnt() << ") failed";
            return -3;
        }
        vals_ = cols_.selectUInts(masks_);
        if (vals_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << cols_.name() << "->selectUInts("
                << masks_.cnt() << ") failed";
            return -4;
        }
        nrows = ibis::util::sortMerge
            (*static_cast<array_t<uint32_t>*>(valr_), *orderr_,
             *static_cast<array_t<uint32_t>*>(vals_), *orders_,
             delta1_, delta2_);
        break;}
    case ibis::LONG: {
        valr_ = colr_.selectLongs(maskr_);
        if (valr_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << colr_.name() << "->selectLongs("
                << maskr_.cnt() << ") failed";
            return -3;
        }
        vals_ = cols_.selectLongs(masks_);
        if (vals_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << cols_.name() << "->selectLongs("
                << masks_.cnt() << ") failed";
            return -4;
        }
        nrows = ibis::util::sortMerge
            (*static_cast<array_t<int64_t>*>(valr_), *orderr_,
             *static_cast<array_t<int64_t>*>(vals_), *orders_,
             delta1_, delta2_);
        break;}
    case ibis::ULONG: {
        valr_ = colr_.selectULongs(maskr_);
        if (valr_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << colr_.name() << "->selectULongs("
                << maskr_.cnt() << ") failed";
            return -3;
        }
        vals_ = cols_.selectULongs(masks_);
        if (vals_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << cols_.name() << "->selectULongs("
                << masks_.cnt() << ") failed";
            return -4;
        }
        nrows = ibis::util::sortMerge
            (*static_cast<array_t<uint64_t>*>(valr_), *orderr_,
             *static_cast<array_t<uint64_t>*>(vals_), *orders_,
             delta1_, delta2_);
        break;}
    case ibis::FLOAT: {
        valr_ = colr_.selectFloats(maskr_);
        if (valr_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << colr_.name() << "->selectFloats("
                << maskr_.cnt() << ") failed";
            return -3;
        }
        vals_ = cols_.selectFloats(masks_);
        if (vals_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << cols_.name() << "->selectFloats("
                << masks_.cnt() << ") failed";
            return -4;
        }
        nrows = ibis::util::sortMerge
            (*static_cast<array_t<float>*>(valr_), *orderr_,
             *static_cast<array_t<float>*>(vals_), *orders_, delta1_, delta2_);
        break;}
    case ibis::DOUBLE: {
        valr_ = colr_.selectDoubles(maskr_);
        if (valr_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << colr_.name() << "->selectDoubles("
                << maskr_.cnt() << ") failed";
            return -3;
        }
        vals_ = cols_.selectDoubles(masks_);
        if (vals_ == 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange::count(" << desc_
                << ") call to " << cols_.name() << "->selectDoubles("
                << masks_.cnt() << ") failed";
            return -4;
        }
        nrows = ibis::util::sortMerge
            (*static_cast<array_t<double>*>(valr_), *orderr_,
             *static_cast<array_t<double>*>(vals_), *orders_, delta1_, delta2_);
        break;}
    }
    LOGGER(ibis::gVerbose > 2)
        << "jRange::count(" << desc_ << ") found " << nrows
        << " hit" << (nrows>1?"s":"");
    return nrows;
} // ibis::jRange::count
Пример #11
0
/// Constructor.
ibis::jRange::jRange(const ibis::part& partr, const ibis::part& parts,
                     const ibis::column& colr, const ibis::column& cols,
                     double delta1, double delta2,
                     const ibis::qExpr* condr, const ibis::qExpr* conds,
                     const ibis::selectClause* sel, const ibis::fromClause* frm,
                     const char* desc)
    : sel_(sel ? new ibis::selectClause(*sel) : 0),
      frm_(frm ? new ibis::fromClause(*frm) : 0),
      partr_(partr), parts_(parts), colr_(colr), cols_(cols),
      delta1_(delta1), delta2_(delta2), orderr_(0), orders_(0),
      valr_(0), vals_(0), nrows(-1) {
    if (desc == 0 || *desc == 0) { // build a description string
        std::ostringstream oss;
        oss << "From " << partr.name() << " Join " << parts.name()
            << " On " << delta1 << " <= " << partr.name() << '.'
            << colr.name() << " - " << parts.name() << '.' << cols.name()
            << " <= " << delta2 << " Where ...";
        desc_ = oss.str();
    }
    else {
        desc_ = desc;
    }
    int ierr;
    if (condr != 0) {
        ibis::countQuery que(&partr);
        ierr = que.setWhereClause(condr);
        if (ierr < 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange(" << desc_ << ") could apply "
                << condr << " on partition " << partr.name()
                << ", ierr = " << ierr;
            throw "jRange::ctor failed to apply conditions on partr"
                IBIS_FILE_LINE;
        }
        ierr = que.evaluate();
        if (ierr < 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange(" << desc_
                << ") could not evaluate " << que.getWhereClause()
                << " on partition " << partr.name() << ", ierr = " << ierr;
            throw "jRange::ctor failed to evaluate constraints on partr"
                IBIS_FILE_LINE;
        }
        maskr_.copy(*que.getHitVector());
    }
    else {
        colr.getNullMask(maskr_);
    }
    if (conds != 0) {
        ibis::countQuery que(&parts);
        ierr = que.setWhereClause(conds);
        if (ierr < 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange(" << desc_ << ") could apply "
                << conds << " on partition " << parts.name()
                << ", ierr = " << ierr;
            throw "jRange::ctor failed to apply conditions on parts"
                IBIS_FILE_LINE;
        }
        ierr = que.evaluate();
        if (ierr < 0) {
            LOGGER(ibis::gVerbose > 1)
                << "Warning -- jRange(" << desc_
                << ") could not evaluate " << que.getWhereClause()
                << " on partition " << parts.name() << ", ierr = " << ierr;
            throw "jRange::ctor failed to evaluate constraints on parts"
                IBIS_FILE_LINE;
        }
        masks_.copy(*que.getHitVector());
    }
    else {
        cols.getNullMask(masks_);
    }
    LOGGER(ibis::gVerbose > 2)
        << "jRange(" << desc_ << ") construction complete";
} // ibis::jRange::jRange
Пример #12
0
ibis::table*
ibis::jRange::fillResult(size_t nrows, double delta1, double delta2,
                         const std::string &desc,
                         const ibis::array_t<T>& rjcol,
                         const ibis::table::typeArray& rtypes,
                         const ibis::table::bufferArray& rbuff,
                         const ibis::array_t<T>& sjcol,
                         const ibis::table::typeArray& stypes,
                         const ibis::table::bufferArray& sbuff,
                         const ibis::table::stringArray& tcname,
                         const std::vector<uint32_t>& tcnpos) {
    if (nrows > (rjcol.size() * sjcol.size()) ||
        rtypes.size() != rbuff.size() || stypes.size() != sbuff.size() ||
        tcname.size() != rtypes.size() + stypes.size() ||
        tcnpos.size() != tcname.size()) {
        LOGGER(ibis::gVerbose > 1)
            << "Warning -- jRange::fillResult can not proceed due "
            "to invalid arguments";
        return 0;
    }
    std::string tn = ibis::util::shortName(desc.c_str());
    if (nrows == 0 || rjcol.empty() || sjcol.empty() ||
        (stypes.empty() && rtypes.empty()))
        return new ibis::tabula(tn.c_str(), desc.c_str(), nrows);

    ibis::table::bufferArray tbuff(tcname.size());
    ibis::table::typeArray   ttypes(tcname.size());
    IBIS_BLOCK_GUARD(ibis::table::freeBuffers, ibis::util::ref(tbuff),
                     ibis::util::ref(ttypes));
    try {
        // allocate enough space for the output table
        for (size_t j = 0; j < tcname.size(); ++ j) {
            if (tcnpos[j] < rtypes.size()) {
                ttypes[j] = rtypes[tcnpos[j]];
                tbuff[j] = ibis::table::allocateBuffer
                    (rtypes[tcnpos[j]], nrows);
            }
            else if (tcnpos[j] < rtypes.size()+stypes.size()) {
                ttypes[j] = stypes[tcnpos[j]-rtypes.size()];
                tbuff[j] = ibis::table::allocateBuffer
                    (stypes[tcnpos[j]-rtypes.size()], nrows);
            }
            else { // tcnpos is out of valid range
                ttypes[j] = ibis::UNKNOWN_TYPE;
                tbuff[j] = 0;
                LOGGER(ibis::gVerbose > 0)
                    << "Warning -- jRange::fillResult detects an "
                    "invalid tcnpos[" << j << "] = " << tcnpos[j]
                    << ", should be less than " << rtypes.size()+stypes.size();
                return 0;
            }
        }
    }
    catch (...) {
        LOGGER(ibis::gVerbose > 0)
            << "Warning -- jRange::fillResult failed to allocate "
            "sufficient memory for " << nrows << " row" << (nrows>1?"s":"")
            << " and " << rtypes.size()+stypes.size()
            << " column" << (rtypes.size()+stypes.size()>1?"s":"");
        return 0;
    }

    size_t tind = 0; // row index into the resulting table
    uint32_t ir0 = 0;
    uint32_t ir1 = 0;
    uint32_t is = 0;
    const uint32_t nr = rjcol.size();
    const uint32_t ns = sjcol.size();
    while (ir0 < nr && is < ns) {
        while (ir0 < nr && rjcol[ir0] < sjcol[is]+delta1)
            ++ ir0;
        ir1 = (ir1>=ir0?ir1:ir0);
        while (ir1 < nr && rjcol[ir1] <= sjcol[is]+delta2)
            ++ ir1;
        if (ir1 > ir0) { // found matches
            size_t is0 = is;
            while (is < ns && sjcol[is] == sjcol[is0])
                ++ is;
            LOGGER(ibis::gVerbose > 5)
                << "DEBUG -- jRange::fillResult: ir0=" << ir0 << ", ir1="
                << ir1 << ", is0=" << is0 << ", is1=" << is << ", rjcol["
                << ir0 << "]=" << rjcol[ir0] << ", rjcol[" << ir1 << "]="
                << rjcol[ir1] << ", sjcol[" << is0 << "]=" << sjcol[is0]
                << ", sjcol[" << is << "]=" << sjcol[is];
            for (size_t jr = ir0; jr < ir1; ++ jr) {
                for (size_t js = is0; js < is; ++ js) {
                    for (size_t jt = 0; jt < tcnpos.size(); ++ jt) {
                        if (tcnpos[jt] < rbuff.size()) {
                            ibis::bord::copyValue(rtypes[tcnpos[jt]],
                                                  tbuff[jt], tind,
                                                  rbuff[tcnpos[jt]], jr);
                        }
                        else {
                            ibis::bord::copyValue
                                (stypes[tcnpos[jt]-rtypes.size()],
                                 tbuff[jt], tind,
                                 sbuff[tcnpos[jt]-rtypes.size()], js);
                        }
                    } // jt
                    ++ tind;
                } // js
            } // jr
        }
        else {
            ++ is;
        }
    } // while ...
    if (tind != nrows) {
        LOGGER(ibis::gVerbose >= 0)
            << "Warning -- jRange::fillResult expected to produce "
            << nrows << " row" << (nrows>1?"s":"") << ", but produced "
            << tind << " instead";
        return 0;
    }

    LOGGER(ibis::gVerbose > 3)
        << "jRange(" << desc << ")::fillResult produced " << tind
        << " row" << (tind>1?"s":"") << " for \"" << typeid(T).name()
        << '[' << rjcol.size() << "] - " << typeid(T).name()
        << '[' << sjcol.size() << "] between " << delta1 << " and " << delta2
        << '\"';
    return new ibis::bord(tn.c_str(), desc.c_str(), nrows,
                          tbuff, ttypes, tcname);
} // ibis::jRange::fillResult
return_type main(int argc, 		/*argument count*/
                 char* argv[]	/*argument array*/
                )
{
    int err_no = ZERO;
    char *sdp_msg_string = NULL;
    sdp_session_t *my_session = NULL;
    char buff[FILE_SIZE];

    if(argc  !=  ARGC)
    {
        error_handling(ERR_MAJOR, SDP_ARGC_ERROR,
                       "Usage : <executable> <file name>");
        exit(EXIT_FAILURE);
    }

    memset(buff, 0, FILE_SIZE);
    if(gethostname(buff, FILE_SIZE) != SUCCESS)
    {
        error_handling(ERR_MAJOR, SDP_GETHOSTNAME_ERROR,
                       "sdp_server_main : Gethostname error");
        exit(EXIT_FAILURE);
    }
    printf("Hostname is : %s\n ", buff);
    LOGGER(LOG_CRITICAL, "Start of sdp_hash_append");
    if(sdp_hash_append(argv[1]) != SUCCESS)
    {
        error_handling(ERR_MAJOR, SDP_FILE_HASH_ERROR,
                       "sdp_server_main : sdp_hash_append error.");
        LOGGER(LOG_CRITICAL, "End of sdp_hash_append");
        exit(EXIT_FAILURE);
    }
    LOGGER(LOG_CRITICAL, "End of sdp_hash_append");

    log_level = LOG_CRITICAL;
    log_max = LOG_MAX;
    if(NULL == strncpy(log_file,"sdp_server.log", strlen("sdp_server.log") + 1))
    {
        error_handling(ERR_MAJOR, SDP_STRNCPY_ERROR,
                       "sdp_server_main : strncpy error.");
        exit(EXIT_FAILURE);
    }
    if(NULL == strncpy(program_name,"main_server.c", strlen("main_server.c") + 1))
    {
        error_handling(ERR_MAJOR, SDP_STRNCPY_ERROR,
                       "sdp_server_main : strncpy error.");
        exit(EXIT_FAILURE);
    }

    LOGGER(LOG_CRITICAL, "Start of sdp_checker");
    err_no=sdp_checker(inputfile);
    if(err_no  != SUCCESS)
    {
        error_handling(ERR_MAJOR, SDP_CHECKER_ERROR,
                       "sdp_server_main : sdp_checker error.");
        LOGGER(LOG_CRITICAL, "End of sdp_checker");
        exit(EXIT_FAILURE);
    }
    LOGGER(LOG_CRITICAL, "End of sdp_checker");

    my_session = (sdp_session_t *)calloc(1, sizeof(sdp_session_t));

    if(NULL == my_session)
    {
        error_handling(ERR_MAJOR, SDP_STRING_MALLOC_ERROR,
                       "sdp_server_main : malloc error.");
        exit(EXIT_FAILURE);
    }

    LOGGER(LOG_CRITICAL, "Start of sdp_populate_message");
    if(sdp_populate_message(my_session, inputfile) != SUCCESS)
    {
        error_handling(ERR_MAJOR, SDP_POPULATE_ERROR,
                       "sdp_server_main : sdp_populate_message error.");
        LOGGER(LOG_CRITICAL, "End of sdp_populate_message");
        myfree(my_session);
        exit(EXIT_FAILURE);
    }
    LOGGER(LOG_CRITICAL, "End of sdp_populate_message");

    LOGGER(LOG_CRITICAL, "Start of sdp_session_to_string");
    sdp_msg_string = sdp_session_to_str(my_session, &err_no);
    if(sdp_msg_string == NULL)
    {
        error_handling(ERR_MAJOR, SDP_SESSION_TO_STR_ERROR,
                       "sdp_server_main : sdp_session_to_str error.");
        LOGGER(LOG_CRITICAL, "End of sdp_session_to_string");
        myfree(my_session);
        exit(EXIT_FAILURE);
    }
    LOGGER(LOG_CRITICAL, "End of sdp_session_to_string");
    myfree(my_session);

    printf("%s\n",sdp_msg_string);

    LOGGER(LOG_CRITICAL, "Start of sdp_sender");
    if(sdp_server_sender(sdp_msg_string) != SUCCESS)
    {
        error_handling(ERR_MAJOR, SDP_SENDER_ERROR,
                       "sdp_server_main : sdp_server_sender error.");
        LOGGER(LOG_CRITICAL, "End of sdp_server_sender");
        free(sdp_msg_string);
        exit(EXIT_FAILURE);
    }
    LOGGER(LOG_CRITICAL, "End of sdp_server_sender");

    free(sdp_msg_string);
    return SUCCESS;
}
Пример #14
0
ibis::math::term* ibis::selectClause::addRecursive(ibis::math::term*& tm) {
    if (tm == 0) return tm;

    switch (tm->termType()) {
    default:
    case ibis::math::NUMBER:
    case ibis::math::STRING:
        break; // nothing to do
    case ibis::math::VARIABLE: {
        ibis::selectClause::variable *var =
            dynamic_cast<ibis::selectClause::variable *>(tm);
        if (var == 0) { // a bare variable
            const char* vname =
                static_cast<ibis::math::variable*>(tm)->variableName();
            if (ordered_.find(vname) == ordered_.end()) {
                const unsigned pos = atms_.size();
                aggr_.push_back(ibis::selectClause::NIL_AGGR);
                atms_.push_back(tm->dup());
                ordered_[vname] = pos;

                LOGGER(ibis::gVerbose > 5)
                    << "selectClause::addRecursive -- adding term "
                    << pos << ": " << vname;
            }
        }
        break;}
    case ibis::math::STDFUNCTION1:
    case ibis::math::CUSTOMFUNCTION1:
    case ibis::math::STRINGFUNCTION1: {
        ibis::math::term *nxt =
            reinterpret_cast<ibis::math::term*>(tm->getLeft());
        if (nxt == 0) {
            return nxt;
        }
        else if (hasAggregation(nxt)) {
            ibis::math::term *tmp = addRecursive(nxt);
            if (tmp != nxt)
                tm->getLeft() = tmp;
        }
        else {
            const unsigned pos = atms_.size();
            aggr_.push_back(ibis::selectClause::NIL_AGGR);
            atms_.push_back(tm);
            LOGGER(ibis::gVerbose > 5)
                << "selectClause::addRecursive -- adding term "
                << pos << ": " << aggDescription(pos);

            std::ostringstream oss;
            oss << "__" << std::hex << pos;
            ordered_[oss.str()] = pos;
            return new ibis::selectClause::variable(oss.str().c_str(), this);
        }
        break;}
    case ibis::math::OPERATOR:
    case ibis::math::STDFUNCTION2:
    case ibis::math::CUSTOMFUNCTION2:
    case ibis::math::STRINGFUNCTION2: {
        ibis::math::term *left =
            reinterpret_cast<ibis::math::term*>(tm->getLeft());
        ibis::math::term *right =
            reinterpret_cast<ibis::math::term*>(tm->getRight());
        if (left == 0) {
            if (right == 0) {
                return 0;
            }
            else if (dynamic_cast<ibis::selectClause::variable*>(right) == 0) {
                tm->getRight() = addRecursive(right);
            }
        }
        else if (dynamic_cast<ibis::selectClause::variable*>(left) != 0) {
            if (dynamic_cast<ibis::selectClause::variable*>(right) == 0) {
                tm->getRight() = addRecursive(right);
            }
        }
        else if (dynamic_cast<ibis::selectClause::variable*>(right) != 0) {
            tm->getLeft() = addRecursive(left);
        }
        else if (hasAggregation(tm)) {
            tm->getLeft() = addRecursive(left);
            tm->getRight() = addRecursive(right);
        }
        else {
            const unsigned pos = atms_.size();
            aggr_.push_back(ibis::selectClause::NIL_AGGR);
            atms_.push_back(tm);
            LOGGER(ibis::gVerbose > 5)
                << "selectClause::addRecursive -- adding term "
                << pos << ": " << aggDescription(pos);

            std::ostringstream oss;
            oss << "__" << std::hex << pos;
            ordered_[oss.str()] = pos;
            return new ibis::selectClause::variable(oss.str().c_str(), this);
        }
        break;}
    }
    return tm;
} // ibis::selectClause::addRecursive
void stub_call(int log_level,char* message)
{
	LOGGER(log_level, message);
}
Пример #16
0
bool EflResources::copyResource( Evas_Object* const _image
                               , std::string const& _path
                               , bool const         _keep_aspect
                               , int                _width
                               , int                _height ) const
{
    bool result( true );
    Evas_Object* object = nullptr;

    if( 0 != preloaded_images__.count( _path ) )
        object = preloaded_images__.find( _path )->second;
    else
    {
        object = preloaded_images__.find( IMG_DIR "/placeholder.png" )->second;
        LOGGER( "Could not find file among preloaded images: " + _path );
        result = false;
    }

    int src_w = 0;
    int src_h = 0;

    evas_object_image_size_get( object
                              , &src_w
                              , &src_h );
    evas_object_image_size_set( _image
                              , src_w
                              , src_h );
    evas_object_image_alpha_set( _image
                               , evas_object_image_alpha_get( object ) );
    evas_object_image_data_set( _image
                              , evas_object_image_data_get( object
                                                          , 0 ) );


    if( _keep_aspect )
    {
        if( 0 == _width || 0 == _height )
        {
            evas_object_geometry_get( _image
                                    , nullptr
                                    , nullptr
                                    , &_width
                                    , &_height );
        }

        int new_w = 0;
        int new_h = 0;
        Utility::calculateImageSize( _width
                                   , _height
                                   , src_w
                                   , src_h
                                   , new_w
                                   , new_h );
        evas_object_resize( _image
                          , new_w
                          , new_h );
    }

    evas_object_image_pixels_dirty_set( _image
                                      , 1 );
    return result;
}
Пример #17
0
/**
 * Accepts a connection and returns a socket
 * for the recently accepted connection
 *
 * @param listen_socket         socket in a 'listening' state
 * @param accept_socket         pointer by reference to store the accepted connection socket
 *
 * @return                      TRUE if connection was accepted, otherwise FALSE
 */
int socket_accept(SOCKET_T * listen_socket, SOCKET_T ** accept_socket) {

  int socket_handle;
  SOCKET_T* new_acc_socket;
  struct sockaddr sa_client;
  unsigned int sa_client_size = sizeof(sa_client);
  char buffer[1024];

  MUTEX_LOCK(listen_socket->mutex);

  if ( listen_socket != NULL ) {

    // Allocates space for the socket structure
    new_acc_socket = malloc(sizeof( SOCKET_T ));
    memset(new_acc_socket, 0x00, sizeof( SOCKET_T ));

    // Accepts connection
    socket_handle = accept(listen_socket->handle, (struct sockaddr *) &sa_client, &sa_client_size);
    if (socket_handle == -1) {

      if ( errno == EAGAIN || errno == EWOULDBLOCK ) {

        // Returns invalid, no connections pending
        free(new_acc_socket);
        MUTEX_UNLOCK(listen_socket->mutex);
        return FALSE;

      } else {
        
        sprintf(buffer, "accept failed with error: %d\n", errno);
        LOGGER(__FUNCTION__, buffer);

        free(new_acc_socket);
        MUTEX_UNLOCK(listen_socket->mutex);
        return FALSE;
      }

    } else {

      // Stores the new socket handle in the structure
      new_acc_socket->handle = socket_handle;

      // Copies the structure to the output parameter
      *accept_socket = new_acc_socket;

      // Creates mutex for the new socket
      (*accept_socket)->mutex = (MUTEX_T*)malloc(sizeof(struct _mutex_t) );
      memset( (*accept_socket)->mutex, 0x00, sizeof(struct _mutex_t));
      MUTEX_CREATE( &((*accept_socket)->mutex) );

      MUTEX_UNLOCK(listen_socket->mutex);

      return TRUE;
    }
  }

  MUTEX_UNLOCK(listen_socket->mutex);

  return FALSE;

}
Пример #18
0
int
offline_capture(FILE *fcapture) {
    struct bpf_program bpf;
    char errbuf[PCAP_ERRBUF_SIZE];
    char filter[300];
    char ports_str[256];
    char **ports;
    int r, n_ports;

    pcap = pcap_fopen_offline(fcapture, errbuf);
    if (!pcap) {
        LOGGER(ERROR, "pcap: %s\n", errbuf);
        return 1;
        
    }
    
    if(port) {
        int i, n = 0 ;
        ports = split_string(port, strlen(port), ",", 1, &n_ports);
        if(n_ports > 10) {
            LOGGER(ERROR, "it's unscientific to listen so many ports.\n", errbuf);
            return 1;
        }
       
        n = snprintf(ports_str, 256, "tcp port %s", ports[0]);
        
        for(i = 1; i < n_ports; i++) {
            n += snprintf(ports_str + n, 256, " or tcp port %s", ports[i]);
        }
        split_string_free(ports, n_ports);
    }

    // Capture only TCP
    if (global_options.server && n_ports) {
        sprintf(filter, "host %s and (%s)", global_options.server, ports_str);
    } else if (global_options.server && !n_ports) {
        sprintf(filter, "host %s", global_options.server);
    } else if (!global_options.server && n_ports) {
        sprintf(filter, "(%s)", ports_str);
    } else {
        sprintf(filter, "tcp");
    }

    if (pcap_compile(pcap, &bpf, filter, 1, 0)) {
        LOGGER(ERROR, "pcap: %s\n", pcap_geterr(pcap));
        return 1;
        
    }
    
    if (pcap_setfilter(pcap, &bpf)) {
        LOGGER(ERROR, "pcap: %s\n", pcap_geterr(pcap));
        return 1;
        
    }
    
    // The -1 here stands for "infinity"
    r = pcap_loop(pcap, -1, process_packet, (unsigned char *) pcap);
    if (r == -1) {
        LOGGER(ERROR, "pcap: %s\n", pcap_geterr(pcap));
        return 1;
        
    }
    
    return 1;
    
}
Пример #19
0
/**
 * Checks if a group of sockets is available for read/write
 * 
 * Upon return the lists will be already updated, having removed de nodes correponding
 * to sockets that were not available for the requested operations (read/write)
 * 
 * @param timeout               timeout, or 0 for blocking
 * @param read_s                pointer to a socket list to check for read availability, or NULL if does not apply
 * @param write_s               pointer to a socket list to check for write availability, or NULL if does not apply
 *
 * @return                      TRUE, o FALSE si la operacion dio error o timeout
 */
int socket_select_multiple(int timeout, list_t* read_s, list_t* write_s ) {

  list_node_t * seeker = NULL;
  list_node_t * remover = NULL;
  SOCKET_T * ptr_socket = NULL;
  fd_set* readfds = NULL;
  fd_set* writefds = NULL;
  struct timeval tval_timeout;
  int lockstate;
  char buffer[1024];
  int res;

  // configures timeout
  tval_timeout.tv_sec = timeout;
  tval_timeout.tv_usec = 0;

  // Reserva memoria para los sets
  readfds = (fd_set*)malloc(sizeof(fd_set));
  writefds = (fd_set*)malloc(sizeof(fd_set));

  // Initializes sets
  FD_ZERO(readfds);
  FD_ZERO(writefds);

  // Iterates through the read sockets list
  seeker = read_s->first;
  while ( seeker != NULL) {

    ptr_socket = seeker->content;
    if (ptr_socket != NULL) {

      // Puts a lock on the socket's mutex
      MUTEX_LOCK(ptr_socket->mutex);

      // Adds socket to the read set
      FD_SET( ptr_socket->handle, readfds);

    }

    seeker = seeker->next;
  
  }

  // Iterates through the write sockets list
  seeker = write_s->first;
  while ( seeker != NULL) {

    ptr_socket = seeker->content;
    if (ptr_socket != NULL) {

      // If it is not locked
      lockstate = MUTEX_IS_LOCKED(ptr_socket->mutex); 
      if ( lockstate == FALSE ) {

        // Puts a lock on the socket's mutex
        MUTEX_LOCK(ptr_socket->mutex);

      }

      // Adds socket to the write set
      FD_SET( ptr_socket->handle, writefds );
    
    }

    seeker = seeker->next;
  
  }

  // Calls the select function with the generated sets
  res = select(0, readfds, writefds, NULL, &tval_timeout);
  if (res == -1) {

    sprintf(buffer, "select failed with error: %d\n", errno);
    LOGGER(__FUNCTION__, buffer);
    
    // Frees memory
    free(readfds);
    free(writefds);

    return FALSE;

  }
  
  //
  // Goes through the socket lists checking for the available ones and
  // removing the ones that are not available
  //

  //
  // Goes through the read list
  //
  seeker = read_s->first;
  while ( seeker != NULL) {

    ptr_socket = seeker->content;
    if (ptr_socket != NULL) {

      // If a socket is not available for reading
      if ( !FD_ISSET( ptr_socket->handle, readfds) ) {
        
        // Removes the socket from the list
        remover = seeker;
        seeker = seeker->next;

        LIST_REMOVE(read_s, remover, NULL);

        // Removes the lock on the mutex
        MUTEX_UNLOCK(ptr_socket->mutex);

        // Moves to the next
        continue;

      } else {

        // Removes the lock on the mutex
        MUTEX_UNLOCK(ptr_socket->mutex);
      
      }
    
    }

    // Moves to the next
    seeker = seeker->next;
  }

  //
  // Goes through the write list
  //
  seeker = write_s->first;
  while ( seeker != NULL) {

    ptr_socket = seeker->content;
    if (ptr_socket != NULL) {

      // If it is not locked
      lockstate = MUTEX_IS_LOCKED(ptr_socket->mutex); 
      if ( lockstate == FALSE ) {

        // Puts a lock on the socket's mutex
        MUTEX_LOCK(ptr_socket->mutex);
      
      }

      // If socket is not available for writing
      if ( !FD_ISSET( ptr_socket->handle, writefds) ) {

        // Removes the socket from the list
        remover = seeker;
        seeker = seeker->next;

        LIST_REMOVE(write_s, remover, NULL);

        // Removes the lock on the socket's mutex
        MUTEX_UNLOCK(ptr_socket->mutex);
        
        // Moves to the next
        continue;
      
      } else {
        
        // Removes the lock on the socket's mutex      
        MUTEX_UNLOCK(ptr_socket->mutex);        
      
      }

    }

    // Moves to the next
    seeker = seeker->next;
  
  }

  // Frees memory
  free(readfds);
  free(writefds);

  return TRUE;

}
Пример #20
0
void *
capture(void *arg) {
    struct bpf_program bpf;
    char errbuf[PCAP_ERRBUF_SIZE];
    char filter[300];
    char ports_str[256];
    char **ports;
    int r, n_ports;

    // Second argument 0 stands for non-promiscuous mode
    pcap = pcap_open_live(global_options.interface, CAPTURE_LENGTH, 0, READ_TIMEOUT, errbuf);
    if (!pcap) {
        LOGGER(ERROR, "pcap: %s\n", errbuf);
        return NULL;
        
    }
    
    if(port) {
        int i, n = 0 ;
        ports = split_string(port, strlen(port), ",", 1, &n_ports);
        if(n_ports > 10) {
            LOGGER(ERROR, "it's unscientific to listen so many ports.\n", errbuf);
            return NULL;
        }
       
        n = snprintf(ports_str, 256, "tcp port %s", ports[0]);
        
        for(i = 1; i < n_ports; i++) {
            n += snprintf(ports_str + n, 256, " or tcp port %s", ports[i]);
        }
        split_string_free(ports, n_ports);
    }

    // Capture only TCP
    if (global_options.server && n_ports) {
        sprintf(filter, "host %s and (%s)", global_options.server, ports_str);
    } else if (global_options.server && !n_ports) {
        sprintf(filter, "host %s", global_options.server);
    } else if (!global_options.server && n_ports) {
        sprintf(filter, " (%s)", ports_str);
    } else {
        sprintf(filter, "tcp");
    }

    if (pcap_compile(pcap, &bpf, filter, 1, 0)) {
        LOGGER(ERROR, "pcap: %s\n", pcap_geterr(pcap));
        return NULL;
        
    }
    
    if (pcap_setfilter(pcap, &bpf)) {
        LOGGER(ERROR, "pcap: %s\n", pcap_geterr(pcap));
        return NULL;
        
    }
    
    // The -1 here stands for "infinity"
    r = pcap_loop(pcap, -1, process_packet, (unsigned char *) pcap);
    if (r == -1) {
        LOGGER(ERROR, "pcap: %s\n", pcap_geterr(pcap));
        return NULL;
        
    }
    
    return NULL;
    
}
Пример #21
0
/**
 ** \~french
 * \brief Fonction principale de l'outil createNodata
 * \details Tout est contenu dans cette fonction. Le "cropage" se fait grâce à la classe TiffNodataManager, et le tuilage / compression est géré par TiledTiffWriter
 * \param[in] argc nombre de paramètres
 * \param[in] argv tableau des paramètres
 * \return code de retour, 0 en cas de succès, -1 sinon
 ** \~english
 * \brief Main function for tool createNodata
 * \details All instrcutions are in this function. the crop is handled by the class TiffNodataManager and TiledTiffWriter make image tiled and compressed.
 * \param[in] argc parameters number
 * \param[in] argv parameters array
 * \return return code, 0 if success, -1 otherwise
 */
int main ( int argc, char **argv ) {

    char* input = 0, *output = 0;
    int tileWidth = 256, tileHeight = 256;
    Compression::eCompression compression = Compression::NONE;
    bool crop = false;
    bool debugLogger=false;

    /* Initialisation des Loggers */
    Logger::setOutput ( STANDARD_OUTPUT_STREAM_FOR_ERRORS );

    Accumulator* acc = new StreamAccumulator();
    Logger::setAccumulator ( INFO , acc );
    Logger::setAccumulator ( WARN , acc );
    Logger::setAccumulator ( ERROR, acc );
    Logger::setAccumulator ( FATAL, acc );

    std::ostream &logw = LOGGER ( WARN );
    logw.precision ( 16 );
    logw.setf ( std::ios::fixed,std::ios::floatfield );

    // Récupération des paramètres
    for ( int i = 1; i < argc; i++ ) {
        if ( !strcmp ( argv[i],"-crop" ) ) {
            crop = true;
            continue;
        }
        if ( argv[i][0] == '-' ) {
            switch ( argv[i][1] ) {
                case 'h': // help
                    usage();
                    exit ( 0 );
                case 'd': // debug logs
                    debugLogger = true;
                    break;
                case 'c': // compression
                    if ( ++i == argc ) { error ( "Error in -c option", -1 ); }
                    if ( strncmp ( argv[i], "none",4 ) == 0 || strncmp ( argv[i], "raw",3 ) == 0 ) {
                        compression = Compression::NONE;
                    } else if ( strncmp ( argv[i], "png",3 ) == 0 ) {
                        compression = Compression::PNG;
                    } else if ( strncmp ( argv[i], "jpg",3 ) == 0 ) {
                        compression = Compression::JPEG;
                    } else if ( strncmp ( argv[i], "lzw",3 ) == 0 ) {
                        compression = Compression::LZW;
                    } else if ( strncmp ( argv[i], "zip",3 ) == 0 ) {
                        compression = Compression::DEFLATE;
                    } else if ( strncmp ( argv[i], "pkb",3 ) == 0 ) {
                        compression = Compression::PACKBITS;
                    } else {
                        error ( "Unknown compression : " + string(argv[i]), -1 );
                    }
                    break;
                case 't':
                    if ( i+2 >= argc ) { error("Error in -t option", -1 ); }
                    tileWidth = atoi ( argv[++i] );
                    tileHeight = atoi ( argv[++i] );
                    break;
                default:
                    error ( "Unknown option : " + string(argv[i]) ,-1 );
            }
        } else {
            if ( input == 0 ) input = argv[i];
            else if ( output == 0 ) output = argv[i];
            else { error ( "Argument must specify ONE input file and ONE output file", 2 ); }
        }
    }

    if (debugLogger) {
        // le niveau debug du logger est activé
        Logger::setAccumulator ( DEBUG, acc);
        std::ostream &logd = LOGGER ( DEBUG );
        logd.precision ( 16 );
        logd.setf ( std::ios::fixed,std::ios::floatfield );
    }

    if ( input == 0 || output == 0 ) {
        error ("Argument must specify one input file and one output file", -1);
    }

    FileImageFactory FIF;

    if (crop && compression != Compression::JPEG) {
        LOGGER_WARN("Crop option is reserved for JPEG compression");
        crop = false;
    }

    // For jpeg compression with crop option, we have to remove white pixel, to avoid empty bloc in data
    if ( crop ) {
        LOGGER_DEBUG ( "Open image to read" );
        // On récupère les informations nécessaires pour appeler le nodata manager
        FileImage* tmpSourceImage = FIF.createImageToRead(input);
        int spp = tmpSourceImage->channels;
        int bps = tmpSourceImage->getBitsPerSample();
        SampleFormat::eSampleFormat sf = tmpSourceImage->getSampleFormat();
        delete tmpSourceImage;

        if ( bps == 8 && sf == SampleFormat::UINT ) {
            TiffNodataManager<uint8_t> TNM ( spp, white, true, fastWhite,white );
            if ( ! TNM.treatNodata ( input,input ) ) {
                error ( "Unable to treat white pixels in this image : " + string(input), -1 );
            }
        } else {
            LOGGER_WARN( "Crop option ignored (only for 8-bit integer images) for the image : " << input);
        }
    }

    LOGGER_DEBUG ( "Open image to read" );
    FileImage* sourceImage = FIF.createImageToRead(input);
    if (sourceImage == NULL) {
        error("Cannot read the source image", -1);
    }
    
    if (debugLogger) {
        sourceImage->print();
    }

    Rok4ImageFactory R4IF;
    Rok4Image* rok4Image = R4IF.createRok4ImageToWrite(
        output, BoundingBox<double>(0.,0.,0.,0.), -1, -1, sourceImage->getWidth(), sourceImage->getHeight(), sourceImage->channels,
        sourceImage->getSampleFormat(), sourceImage->getBitsPerSample(), sourceImage->getPhotometric(), compression,
        tileWidth, tileHeight
    );
    
    rok4Image->setExtraSample(sourceImage->getExtraSample());
    
    if (rok4Image == NULL) {
        error("Cannot create the ROK4 image to write", -1);
    }
    
    if (debugLogger) {
        rok4Image->print();
    }

    LOGGER_DEBUG ( "Write" );
    if (rok4Image->writeImage(sourceImage, crop) < 0) {
        error("Cannot write ROK4 image", -1);
    }
    
    LOGGER_DEBUG ( "Clean" );
    // Nettoyage
    delete acc;
    delete sourceImage;
    delete rok4Image;

    return 0;
}
Пример #22
0
int main(int argc, char **argv) 
{
    std::string varPathStr;
    std::string varNameStr1;
    std::string varNameStr2;
    std::string varNameStr3;
    parseArgs(argc, argv);
    std::vector<double> beginList;
    std::vector<double> endList;
    std::vector<double> strideList;
    beginList.resize(dimension);
    endList.resize(dimension);
    strideList.resize(dimension);
    if (datafile.empty() || condstring == 0 || dimension == 0 ) {
	std::cerr << "Usage:\n" << *argv 
		  << " -f data-file-name" 
		  << " -q query-conditions-in-a-single-string"
		  << " -x histogram-dimension"
		  << " -y begin"
		  << " -e end"
		  << " -s stride"
		  << " [-i index-file-name]"
		  << " [-g log-file-name]"
		  << " [-n name-of-variable]"
		  << " [-p path-of-variable]" 
		  << " [-m file model [HDF5(default), H5PART, NETCDF, PNETCDF]"
		  << " [-b use-boundingbox-data-selection]"
		  << " [-v verboseness]"
		  << " [-l mpi-subarray-length]"
	    //<< "\n e.g:   ./histogram -f h5uc-data-index.h5 -q 'px < 0.3' -n y -p TimeStep2 -x 1\n"
		  <<   "\n e.g:   ./histogram -f h5uc-data.h5 -i indexfile -q 'px<0.3 && py>0' -x 2 -n py,pz;"
		  << " -y '0,-0.5;' -s '0.1,0.02;' -e '1,0;' -p TimeStep2\n\n"
		  << "\tFor More detailed usage description and examples, please see file GUIDE"
		  << std::endl;
	return -1;
    }

#ifndef FQ_NOMPI
    MPI_Init(&argc, &argv);
    int mpi_size, mpi_rank;
    MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
#endif

    ibis::gParameters().add(FQ_REPORT_STATISTIC, "true");
	
    ibis::horometer totTimer;
    totTimer.start();

    FQ::FileFormat model = FQ::FQ_HDF5;
    if (fileModel != 0) {
	std::string format = fileModel;
	if (format.compare("HDF5") == 0) {
	    model = FQ::FQ_HDF5;
	} 
	else if (format.compare("H5PART") == 0) {
	    model = FQ::FQ_H5Part;
	} 
	else if (format.compare("NETCDF") == 0) {
	    model = FQ::FQ_NetCDF;
	}
	else if (format.compare("PNETCDF") == 0) {
	    model = FQ::FQ_pnetCDF;
	}
    }

    if (! indexfile.empty()) {
	if (verboseness > 1) 
	    std::cout << "DEBUG: using indexfile \"" << indexfile.c_str() << "\" ... \n";
    }

    if (varPath != 0) {
	if (verboseness > 1)  std::cout << "Debug: use variable path \"" << varPath << "\"\n";
	varPathStr = varPath;
    }
	
    //	std::cout << "varName:" << varName << " begin:" << begin << " end:" << end << " stride:" << stride << std::endl;
    varName1 = strtok(varName, ",;");
    if (dimension>1) varName2 = strtok(NULL, ",;");
    if (dimension>2) varName3 = strtok(NULL, ",;");
    begin1 = atof(strtok(begin, ",;"));
    if (dimension>1) begin2 = atof(strtok(NULL, ",;"));
    if (dimension>2) begin3 = atof(strtok(NULL, ",;"));
    end1 = atof(strtok(end, ",;"));
    if (dimension>1) end2 = atof(strtok(NULL, ",;"));
    if (dimension>2) end3 = atof(strtok(NULL, ",;"));
    stride1 = atof(strtok(stride, ",;"));
    if (dimension>1) stride2 = atof(strtok(NULL, ",;"));
    if (dimension>2) stride3 = atof(strtok(NULL, ",;"));	

    //std::cout << "varName is " << varName1 << ", " << varName2 << ", " << varName3 << std::endl;
    //std::cout << "begin   is " << begin1 << ", " << begin2 << ", " << begin3 << std::endl;
    //std::cout << "end     is " << end1 << ", " << end2 << ", " << end3 << std::endl;
    //std::cout << "stride  is " << stride1 << ", " << stride2 << ", " << stride3 << std::endl;

    if (varName1!=0) varNameStr1 = varName1;
    if (dimension>1 && varName2!=0) varNameStr2 = varName2;
    if (dimension>2 && varName3!=0) varNameStr3 = varName3;
    /*	
	if (mpi_rank==0) {
	unsigned int dims1 = static_cast<uint32_t>(1+floor((end1-begin1)/stride1));
    	unsigned int dims2 = static_cast<uint32_t>(1+floor((end2-begin2)/stride2));
    	unsigned int dims3 = static_cast<uint32_t>(1+floor((end3-begin3)/stride3));
	std::cout << "dims1 * dims2 * dims3 = " << dims1 << " * " << dims2 << " * " << dims3 << std::endl;
	}
    */


    if (logfile.str().empty() != true) {
#ifndef FQ_NOMPI
	logfile << mpi_rank << ".log";
#endif
	if (verboseness > 1) std::cout << "Debug: using logfile \"" << logfile.str().c_str() << "\"\n";
    }
    if (verboseness >1) {
	std::cout << "open the file handler" << std::endl;
    }
    // open the named file
    QueryProcessor* queryProcessor = new QueryProcessor(datafile, model, indexfile, verboseness, "", logfile.str().c_str()); // the file handler

    if (queryProcessor->isValid() == false) {
	if (verboseness > 0) {
	    std::cout << "ERROR: failed to initiate the QueryProcessor object for file \"" 
		      << datafile.c_str() << "\" ...\n";
	    std::cout << "REPORT: failed to complete processing query" << std::endl;
	}
	delete(queryProcessor);
#ifndef FQ_NOMPI
	MPI_Finalize();
#endif
	return -1;
    }

    uint64_t hits = 0;
    // getNumHits
    ibis::horometer timer;
    timer.start();
    hits = queryProcessor->getNumHits(condstring, varPathStr, mpi_dim, mpi_len);
    timer.stop();
    if (verboseness > 1)
	std::cout << "Debug: conditions \"" << condstring 
		  << "\" number of hits " << hits << std::endl;;

    if (hits == 0) {
	if (verboseness > 1) {
	    std::cout << "Warning -- No element is seleteced ==>"
		      << " the rest of the test is skipped!" << std::endl;
	}
	if (verboseness > 0) {
#ifndef FQ_NOMPI        
	    if (mpi_rank==0) {
#endif
		std::cout << "REPORT: successfully completed processing query with " 
			  << hits << " hits" << std::endl;
#ifndef FQ_NOMPI        
	    }
#endif
	}
	delete(queryProcessor);
#ifndef FQ_NOMPI
	MPI_Finalize();
#endif
	return hits;
    }

    // executeQuery
    std::vector<uint64_t> coords;
    std::vector<uint32_t> counts;
    bool herr = true;
    //	if (mpi_rank==0) std::cout<<"histogram starting..."<<std::endl;	
    if (varPath != 0) {
	//coords.reserve(hits*dims.size());
	// hits1 = queryProcessor->executeQuery((char*)condstring, coords, varPathStr, FQ::POINTS_SELECTION, mpi_dim, mpi_len);
	if (dimension==1) { 
	    //counts.assign(static_cast<uint32_t>(1+floor(end1-begin1)/stride1), 0);
	    herr = queryProcessor->get1DHistogram
		((char*) condstring, varNameStr1, varPathStr, begin1, end1, stride1, 
		 counts, mpi_dim, mpi_len);
	} else if (dimension==2) {
	    //			if (mpi_rank==0) std::cout << "in 2Dhistogram" << std::endl;
	    //counts.assign(static_cast<uint32_t>(1+floor(end1-begin1)/stride1)*
	    //          static_cast<uint32_t>(1+floor(end2-begin2)/stride2), 0);
	    herr = queryProcessor->get2DHistogram
		((char*) condstring, varPathStr, varNameStr1, begin1, end1, stride1, 
		 varNameStr2, begin2, end2, stride2, 
		 counts, mpi_dim, mpi_len);       
	    //			if (mpi_rank==0) std::cout << "out 2Dhistogram" << std::endl;
	} else if (dimension==3) {
	    herr = queryProcessor->get3DHistogram
		((char*) condstring, varPathStr, 
		 varNameStr1, begin1, end1, stride1, 
		 varNameStr2, begin2, end2, stride2, 
		 varNameStr3, begin3, end3, stride3,
		 counts, mpi_dim, mpi_len);
	}
	if (! herr) {
	    LOGGER(ibis::gVerbose >= 0)
		<< *argv << " failed to compute the histogram";
	    return -2;
	}

	/************************/
	/*  verify part         */
	/************************/

	if (verification) {

	    //verify the Histogram
	    //if (mpi_rank==0) std::cout << "starting verify the histogram..." << std::endl;
	    uint64_t len = 1;
	    if (len) {
		//std::cout<<"Warning: May use too large memory. Can only check sum.\n";				
	    } else {
		/*				double data[len];
		//
		std::vector<uint32_t> temp_counts;
		temp_counts.assign(static_cast<uint32_t>(1+floor(end-begin)/stride), 0);
		bool verr = true;
		//std::cout << "starting getData...." << std::endl;
		verr = queryProcessor->getData(varNameStr, &data[0], varPathStr);

		#ifndef FQ_NOMPI  
		if (mpi_rank==0) {
		#endif
		if (len<=1000000) {
		//std::cout << "getData success" << std::endl;		
		std::cout << "temp Histogram" << std::endl;
		    
		// copy from fasbit parth.cpp get1DHistogram
		if (len != 0) {
		for (uint32_t i = 0; i < len; ++ i) {
		++ temp_counts[static_cast<uint32_t>((data[i] - begin) / stride)];
		}
		}
		//
		std::cout << "temp Histogram" << std::endl;
		std::cout << "temp_counts.size is "<< temp_counts.size() << std::endl;
		for (int i=0; i<temp_counts.size(); i++) {
		std::cout << "[" << begin+i*stride << ", " << begin+(i+1)*stride << "]:\t" << temp_counts[i] << std::endl;
		}       
		            
		std::cout << "test Histogram" << std::endl;
		std::cout << "counts.size is "<< counts.size() << std::endl;
		for (int i=0; i<counts.size(); i++) {
		std::cout << "[" << begin+i*stride << ", " << begin+(i+1)*stride << "]:\t" << counts[i] << std::endl;
		}
		// verify two histogram vectors	
		if (counts!=temp_counts) {
		std::cout << "ERROR:Vector is not match.Histogram fail." << std::endl;
		} else {
		std::cout << "histogram success" << std::endl;
		}
		}
		#ifndef FQ_NOMPI     
		} 
		#endif
		*/			}
	    //unsigned int hits = 0 ;
	    //hits = queryProcessor->getNumHits(condstring, varPathStr, mpi_dim, mpi_len);
	    uint64_t hits1 = 0;
	    for (int i=0; i<counts.size(); i++) {
		hits1 += counts[i];
	    }
	    if (hits1 != hits) {
		std::cout<<"Error:\tcheck sum failed. Num of Hit is " << hits << ",and histogram number is " << hits1<<std::endl; 
	    } else 
		std::cout<<"verification result is correct.\n";
	}
		
	std::fstream histogramFile;
#ifndef FQ_NOMPI        
	if (mpi_rank==0) {
#endif
	    if (dimension==1) {
		std::fstream file;
		//char fileName[100]="";
		//char path[]="/global/homes/v/vidcina/fq/example/";
		//fileName<<dimension<<"D"<<"histogram["<<begin1<<":"<<stride1<<":"<<end1<<"].out";
		//sprintf(fileName, "%s%d%s%d%s%d%s%d%s", path, dimension, "Dhistogram[", begin1, ":", stride1, ":", end1, "].out");
		//std::string temp="";
		//temp.push_back(fileName.str());
		std::ostringstream fileName;
		fileName << hist_path << "_"<< dimension << "D" << "histogram["
			 << begin1 << ":" << stride1 << ":" <<end1 << "].out";
		std::string str =  fileName.str();
		const char* chr = str.c_str();
		file.open(chr, std::ios::out);
		if ( file.fail() ) {
		    std::cout << str  << std::endl;
		    std::cout << "openFile fail" << std::endl;
		} else {
		    for (int i=0; i<counts.size(); i++) {
			file << begin1+i*stride1 << "\t" << begin1+(i+1)*stride1 << "\t" << counts[i] << std::endl;
		    }
		}
		//histogramFile.close();
	    } else if (dimension==2) {
		std::cout << "2DHistogram "
			  << "Variable1 "<< varName1 << " begin " << begin1
			  << " to " << end1 <<" stride is " << stride1
			  << "Variable2 "<< varName2 << " begin " << begin2
			  << " to " << end2 <<" stride is " << stride2
			  << std::endl ;
		std::cout << "counts.size is "<< counts.size() << std::endl;
		unsigned int imax = static_cast<uint32_t>(1+floor((end1-begin1)/stride1));
		unsigned int jmax = static_cast<uint32_t>(1+floor((end2-begin2)/stride2));
				
		for (unsigned int i=0; i<imax; i++) {
		    for (unsigned int j=0; j<jmax; j++) {
			std::cout << "[" << begin1+i*stride1 << ", "
				  << begin1+(i+1)*stride1 << "), ["
				  << begin2+j*stride2 << ", "
				  << begin2+(j+1)*stride2 << "):\t"
				  << counts[i*jmax+j] << std::endl;
		    }
		}
	    } else if (dimension==3) {
		std::cout << "3DHistogram " 
			  << "Variable1 "<< varName1 << " begin " << begin1
			  << " to " << end1 << " stride is " << stride1 
			  << "Variable2 "<< varName2 << " begin " << begin2
			  << " to " << end2 << " stride is " << stride2 
			  << "Variable3 "<< varName3 << " begin " << begin3
			  << " to " << end2 << " stride is " << stride3
			  << std::endl ;

		std::cout << "counts.size is "<< counts.size() << std::endl;
		unsigned int imax = static_cast<uint32_t>
		    (1+floor((end1-begin1)/stride1));
		unsigned int jmax = static_cast<uint32_t>
		    (1+floor((end2-begin2)/stride2));
		unsigned int kmax = static_cast<uint32_t>
		    (1+floor((end3-begin3)/stride3));
#ifndef FQ_NOMPI
		if (mpi_rank==0 && imax*jmax*kmax!=counts.size()) {
		    std::cout<<"ERROR: counts.size not match."<<std::endl;
		    delete(queryProcessor);
		    MPI_Finalize();
		    return 0;
		}
#endif	
		for (unsigned int i=0; i<imax; i++) {
		    for (unsigned int j=0; j<jmax; j++) {
			for (unsigned int k=0; k<kmax; k++) {
			    if (easyToShow) {
				if (counts[i*jmax*kmax + j*kmax + k]!=0) {
				    std::cout << "[" << begin1+i*stride1 << ", "
					      << begin1+(i+1)*stride1 << "), [" 
					      << begin2+j*stride2 << ", "
					      << begin2+(j+1)*stride2 << "), [" 
					      << begin3+k*stride3 << ", "
					      << begin3+(k+1)*stride3 << "):\t" 
					      << counts[i*jmax*kmax + j*kmax + k]
					      << std::endl;
				}
			    } else {
				std::cout << "[" << begin1+i*stride1 << ", "
					  << begin1+(i+1)*stride1 << "), [" 
					  << begin2+j*stride2 << ", "
					  << begin2+(j+1)*stride2 << "), [" 
					  << begin3+k*stride3 << ", "
					  << begin3+(k+1)*stride3 << "):\t" 
					  << counts[i*jmax*kmax + j*kmax + k]
					  << std::endl;
			    }	
			}
		    }
		}
		std::cout << "successfuly printed histogram" << std::endl;	

	    }

#ifndef FQ_NOMPI        
	} 
#endif
	//		}//end else

    }//end if(!varPath)

 
    //	MPI_Barrier(MPI_COMM_WORLD);
    /*
      if (hits != hits1) 
      {
      std::cout << "Error -- number of hits does not match!" << std::endl;
      std::cout << "REPORT: failed to complete processing query" << std::endl;
      delete(queryProcessor);
      #ifndef FQ_NOMPI
      MPI_Finalize();
      #endif
      return -1;
      }
    */
   
    if (verboseness > 0) {
#ifndef FQ_NOMPI        
	if (mpi_rank==0) {
#endif
	    std::cout << "REPORT: successfully completed get1DHistogram with " 
		      << counts.size() << " histogram size" << std::endl;
#ifndef FQ_NOMPI        
	}
#endif
    }
    delete(queryProcessor);
#ifndef FQ_NOMPI
    MPI_Finalize();
#endif
    totTimer.stop();
    LOGGER(FastQuery::reportTiming())
	<< "Statistic\thistogram::totTimer\t"
        << totTimer.CPUTime() << "\t" << totTimer.realTime()
        << "\t";
    return hits;
} // main
Пример #23
0
// This function reads the data file and records the locations of the values
// in bakMap
void ibis::bak::mapValues(const char* f, ibis::bak::bakMap& bmap) const {
    if (col == 0) return;

    horometer timer;
    if (ibis::gVerbose > 4)
	timer.start();

    const unsigned prec = parsePrec(*col); // the precision of mapped value

    std::string fnm; // name of the data file
    dataFileName(fnm, f);
    if (fnm.empty()) {
	LOGGER(ibis::gVerbose > 0)
	    << "Warning -- bak::mapValues failed to determine the data file "
	    "name from \"" << (f ? f : "") << '"';
	return;
    }

    uint32_t nev;
    ibis::bitvector mask;
    col->getNullMask(mask);
    if (col->partition() != 0)
        nev = col->partition()->nRows();
    else
        nev = mask.size();
    if (nev == 0) return;

    // need to use different types of array_t for different columns
    switch (col->type()) {
    case ibis::TEXT:
    case ibis::UINT: {// unsigned int
	array_t<uint32_t> val;
        if (! fnm.empty())
            ibis::fileManager::instance().getFile(fnm.c_str(), val);
        else
            col->getValuesArray(&val);
	if (val.size() <= 0) {
	    col->logWarning("bak::mapValues", "unable to read %s",
			    fnm.c_str());
	}
	else {
	    bmap.clear();
	    nev = val.size();
	    if (nev > mask.size())
		mask.adjustSize(nev, nev);
	    ibis::bitvector::indexSet iset = mask.firstIndexSet();
	    uint32_t nind = iset.nIndices();
	    const ibis::bitvector::word_t *iix = iset.indices();
	    while (nind) {
		if (iset.isRange()) { // a range
		    uint32_t k = (iix[1] < nev ? iix[1] : nev);
		    for (uint32_t i = *iix; i < k; ++i) {
			double key = ibis::util::coarsen(val[i], prec);
			ibis::bak::grain& grn = bmap[key];
			if (grn.loc == 0)
			    grn.loc = new ibis::bitvector;
			grn.loc->setBit(i, 1);
			if (grn.min > val[i]) grn.min = val[i];
			if (grn.max < val[i]) grn.max = val[i];
		    }
		}
		else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) {
		    // a list of indices
		    for (uint32_t i = 0; i < nind; ++i) {
			uint32_t k = iix[i];
			double key = ibis::util::coarsen(val[k], prec);
			ibis::bak::grain& grn = bmap[key];
			if (grn.loc == 0) grn.loc = new ibis::bitvector;
			grn.loc->setBit(k, 1);
			if (grn.min > val[k]) grn.min = val[k];
			if (grn.max < val[k]) grn.max = val[k];
		    }
		}
		else {
		    for (uint32_t i = 0; i < nind; ++i) {
			uint32_t k = iix[i];
			if (k < nev) {
			    double key = ibis::util::coarsen(val[k], prec);
			    ibis::bak::grain& grn = bmap[key];
			    if (grn.loc == 0)
				grn.loc = new ibis::bitvector;
			    grn.loc->setBit(k, 1);
			    if (grn.min > val[k]) grn.min = val[k];
			    if (grn.max < val[k]) grn.max = val[k];
			}
		    }
		}
		++iset;
		nind = iset.nIndices();
		if (*iix >= nev) nind = 0;
	    } // while (nind)
	}
	break;}
    case ibis::INT: {// signed int
	array_t<int32_t> val;
        if (! fnm.empty())
            ibis::fileManager::instance().getFile(fnm.c_str(), val);
        else
            col->getValuesArray(&val);
	if (val.size() <= 0) {
	    col->logWarning("bak::mapValues", "unable to read %s",
			    fnm.c_str());
	}
	else {
	    nev = val.size();
	    if (nev > mask.size())
		mask.adjustSize(nev, nev);
	    ibis::bitvector::indexSet iset = mask.firstIndexSet();
	    uint32_t nind = iset.nIndices();
	    const ibis::bitvector::word_t *iix = iset.indices();
	    while (nind) {
		if (iset.isRange()) { // a range
		    uint32_t k = (iix[1] < nev ? iix[1] : nev);
		    for (uint32_t i = *iix; i < k; ++i) {
			double key = ibis::util::coarsen(val[i], prec);
			ibis::bak::grain& grn = bmap[key];
			if (grn.loc == 0)
			    grn.loc = new ibis::bitvector;
			grn.loc->setBit(i, 1);
			if (grn.min > val[i]) grn.min = val[i];
			if (grn.max < val[i]) grn.max = val[i];
		    }
		}
		else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) {
		    // a list of indices
		    for (uint32_t i = 0; i < nind; ++i) {
			uint32_t k = iix[i];
			double key = ibis::util::coarsen(val[k], prec);
			ibis::bak::grain& grn = bmap[key];
			if (grn.loc == 0)
			    grn.loc = new ibis::bitvector;
			grn.loc->setBit(k, 1);
			if (grn.min > val[k]) grn.min = val[k];
			if (grn.max < val[k]) grn.max = val[k];
		    }
		}
		else {
		    for (uint32_t i = 0; i < nind; ++i) {
			uint32_t k = iix[i];
			if (k < nev) {
			    double key = ibis::util::coarsen(val[k], prec);
			    ibis::bak::grain& grn = bmap[key];
			    if (grn.loc == 0)
				grn.loc = new ibis::bitvector;
			    grn.loc->setBit(k, 1);
			    if (grn.min > val[k]) grn.min = val[k];
			    if (grn.max < val[k]) grn.max = val[k];
			}
		    }
		}
		++iset;
		nind = iset.nIndices();
		if (*iix >= nev) nind = 0;
	    } // while (nind)
	}
	break;}
    case ibis::FLOAT: {// (4-byte) floating-point values
	array_t<float> val;
        if (! fnm.empty())
            ibis::fileManager::instance().getFile(fnm.c_str(), val);
        else
            col->getValuesArray(&val);
	if (val.size() <= 0) {
	    col->logWarning("bak::mapValues", "unable to read %s",
			    fnm.c_str());
	}
	else {
	    nev = val.size();
	    if (nev > mask.size())
		mask.adjustSize(nev, nev);
	    ibis::bitvector::indexSet iset = mask.firstIndexSet();
	    uint32_t nind = iset.nIndices();
	    const ibis::bitvector::word_t *iix = iset.indices();
	    while (nind) {
		if (iset.isRange()) { // a range
		    uint32_t k = (iix[1] < nev ? iix[1] : nev);
		    for (uint32_t i = *iix; i < k; ++i) {
			double key = ibis::util::coarsen(val[i], prec);
			ibis::bak::grain& grn = bmap[key];
			if (grn.loc == 0)
			    grn.loc = new ibis::bitvector;
			grn.loc->setBit(i, 1);
			if (grn.min > val[i]) grn.min = val[i];
			if (grn.max < val[i]) grn.max = val[i];
		    }
		}
		else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) {
		    // a list of indices
		    for (uint32_t i = 0; i < nind; ++i) {
			uint32_t k = iix[i];
			double key = ibis::util::coarsen(val[k], prec);
			ibis::bak::grain& grn = bmap[key];
			if (grn.loc == 0)
			    grn.loc = new ibis::bitvector;
			grn.loc->setBit(k, 1);
			if (grn.min > val[k]) grn.min = val[k];
			if (grn.max < val[k]) grn.max = val[k];
		    }
		}
		else {
		    for (uint32_t i = 0; i < nind; ++i) {
			uint32_t k = iix[i];
			if (k < nev) {
			    double key = ibis::util::coarsen(val[k], prec);
			    ibis::bak::grain& grn = bmap[key];
			    if (grn.loc == 0)
				grn.loc = new ibis::bitvector;
			    grn.loc->setBit(k, 1);
			    if (grn.min > val[k]) grn.min = val[k];
			    if (grn.max < val[k]) grn.max = val[k];
			}
		    }
		}
		++iset;
		nind = iset.nIndices();
		if (*iix >= nev) nind = 0;
	    } // while (nind)
	}
	break;}
    case ibis::DOUBLE: {// (8-byte) floating-point values
	array_t<double> val;
        if (! fnm.empty())
            ibis::fileManager::instance().getFile(fnm.c_str(), val);
        else
            col->getValuesArray(&val);
	if (val.size() <= 0) {
	    col->logWarning("bak::mapValues", "unable to read %s",
			    fnm.c_str());
	}
	else {
	    nev = val.size();
	    if (nev > mask.size())
		mask.adjustSize(nev, nev);
	    ibis::bitvector::indexSet iset = mask.firstIndexSet();
	    uint32_t nind = iset.nIndices();
	    const ibis::bitvector::word_t *iix = iset.indices();
	    while (nind) {
		if (iset.isRange()) { // a range
		    uint32_t k = (iix[1] < nev ? iix[1] : nev);
		    for (uint32_t i = *iix; i < k; ++i) {
			double key = ibis::util::coarsen(val[i], prec);
			ibis::bak::grain& grn = bmap[key];
			if (grn.loc == 0)
			    grn.loc = new ibis::bitvector;
			grn.loc->setBit(i, 1);
			if (grn.min > val[i]) grn.min = val[i];
			if (grn.max < val[i]) grn.max = val[i];
		    }
		}
		else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) {
		    // a list of indices
		    for (uint32_t i = 0; i < nind; ++i) {
			uint32_t k = iix[i];
			double key = ibis::util::coarsen(val[k], prec);
			ibis::bak::grain& grn = bmap[key];
			if (grn.loc == 0)
			    grn.loc = new ibis::bitvector;
			grn.loc->setBit(k, 1);
			if (grn.min > val[k]) grn.min = val[k];
			if (grn.max < val[k]) grn.max = val[k];
		    }
		}
		else {
		    for (uint32_t i = 0; i < nind; ++i) {
			uint32_t k = iix[i];
			if (k < nev) {
			    double key = ibis::util::coarsen(val[k], prec);
			    ibis::bak::grain& grn = bmap[key];
			    if (grn.loc == 0)
				grn.loc = new ibis::bitvector;
			    grn.loc->setBit(k, 1);
			    if (grn.min > val[k]) grn.min = val[k];
			    if (grn.max < val[k]) grn.max = val[k];
			}
		    }
		}
		++iset;
		nind = iset.nIndices();
		if (*iix >= nev) nind = 0;
	    } // while (nind)
	}
	break;}
    case ibis::CATEGORY: // no need for a separate index
	col->logWarning("bak::mapValues", "no need for binning -- should have "
			"a basic bitmap index already");
	return;
    default:
	col->logWarning("bak::mapValues", "unable to create bins for "
			"this type of column");
	return;
    }

    // make sure all bit vectors are the same size
    for (ibis::bak::bakMap::iterator it = bmap.begin();
	 it != bmap.end(); ++ it) {
	(*it).second.loc->adjustSize(0, nev);
	//	(*it).second.loc->compress();
    }

    // write out the current content
    if (ibis::gVerbose > 2) {
	if (ibis::gVerbose > 4) {
	    timer.stop();
	    col->logMessage("bak::mapValues", "mapped %lu values to %lu "
			    "%u-digit number%s in %g sec(elapsed)",
			    static_cast<long unsigned>(nev),
			    static_cast<long unsigned>(bmap.size()),
			    prec, (bmap.size() > 1 ? "s" : ""),
			    timer.realTime());
	}
	else {
	    col->logMessage("bak::mapValues", "mapped %lu values to %lu "
			    "%lu-digit number%s",
			    static_cast<long unsigned>(nev),
			    static_cast<long unsigned>(bmap.size()), prec,
			    (bmap.size() > 1 ? "s" : ""));
	}
	if (ibis::gVerbose > 6) {
	    ibis::util::logger lg;
	    printMap(lg(), bmap);
	}
    }
} // ibis::bak::mapValues
 void ImageStackDirectoryDatasource::writeToLog()
 {
     LOGGER("Datasource type: ImageStackDirectoryDatasource");
 }
Пример #25
0
/**
 * Thread function for processing an incoming message
 *
 * @param proc_data_arg           data structure with the thread parameters
 */
void process_incoming_request_worker( void * proc_data_arg ) {

  int brecv = 0;

  int selectval = 0;

  char * recbuf = NULL;
  char * incoming_message = NULL;

  int message_complete = FALSE;
  unsigned long message_type = 0;

  long total_bytes_received = 0;
  long incoming_msg_len = 0;
  long var_part_size = 0;
  
  int header_complete = FALSE;

  unsigned long exec_timeout = 0;
  
  PROCESS_DATA_T * proc_data = ( PROCESS_DATA_T * ) proc_data_arg;

  incoming_message = malloc(HEADER_LEN);
  brecv = incoming_msg_len = HEADER_LEN;

  // Allocates initial space for message
  recbuf = malloc(sizeof(char) * HEADER_LEN + VAR_PART_MINIMUM_LEN);

  // Update moment of next timeout
  exec_timeout = (GetTickCount() + gl_timeout);
  
  while (message_complete != TRUE && abort_processes == FALSE) {

    // Evaluates if operation timed out and cancels
    if (GetTickCount() > exec_timeout) {

      LOGGER(__FUNCTION__, "ERROR: Message transfer operation timed out.");
      goto END_PROCESS_INCOMING_REQUEST;
    }

    selectval = SOCKET_SELECT(S_TIMEOUT, proc_data->connection, S_READ);
    if ( selectval == S_READ ) {

      // Attempts to receive the message
      if ( ! SOCKET_RECV(proc_data->connection, &recbuf, &brecv) ) {
    
        // Produces error on fail
        LOGGER(__FUNCTION__, "ERROR: A connection problem occurred while attemting to receive message." );
        break;        
      }

      if ( brecv > 0) {
     
        total_bytes_received += brecv;

        // Updates moment of next timeout
        exec_timeout = GetTickCount() + gl_timeout;

        // Checks if header was previously completed
        if ( header_complete == TRUE ) {
      
          // Copies the new message fragment
          memcpy( &incoming_message[total_bytes_received-brecv], recbuf, brecv );

          // If received message size has reached the total message size
          if ( total_bytes_received == ( HEADER_LEN + var_part_size ) ) {
          
            // Message completed
            message_complete = TRUE;
            break;
          }
          else {

            int bytes_left = incoming_msg_len - total_bytes_received;
            if ( bytes_left < CHUNK_SIZE ) {
              brecv = bytes_left;
            }
            else {
              brecv = CHUNK_SIZE;
            }
          }
        }
        // Checks if header is complete
        else if ( total_bytes_received == HEADER_LEN ) {
      
          // Copies the new fragment of message
          memcpy( &incoming_message[0], recbuf, total_bytes_received );

          // Validates header and gets message type
          message_type = IS_VALID_HEADER( incoming_message, &var_part_size, ( FILE_SND_B + FILE_RCV_B + FILE_DEL_B ) );          
          if ( message_type > 0x00 ) {
        
            header_complete = TRUE;

            if ( var_part_size == 0 ) {
                            
              // If var part size is 0 fails
              LOGGER(__FUNCTION__, "ERROR: Length of variable part cannot be 0." );
              break;
            }
          
            if (var_part_size > CHUNK_SIZE) {
              brecv = CHUNK_SIZE;
            }
            else {
              brecv = var_part_size;
            }

            incoming_msg_len = HEADER_LEN + var_part_size;
            incoming_message = realloc( incoming_message, incoming_msg_len );
            recbuf = realloc( recbuf, incoming_msg_len );
          }
          else {
        
            // If header is not valid fails
            LOGGER(__FUNCTION__, "ERROR: The message does not have a valid header." );
            break;
          }
      
        }

      }

    }

  }

  proc_data->received_message = incoming_message;
  proc_data->received_msg_len = incoming_msg_len;

  // Sends an ACK message to client
  if ( ! process_outgoing_message(proc_data->connection, MESSAGE_ACK, strlen(MESSAGE_ACK)) ) {

    LOGGER(__FUNCTION__, "ERROR: Acknowledgment message could not be sent.");
    goto END_PROCESS_INCOMING_REQUEST;
  }

  // If it is a File Receive message
  if ( message_type == FILE_RCV_B ) {
      
    // Process the message
    process_file_receive( proc_data );    
  }
      
  // If it is a File Send message
  if ( message_type == FILE_SND_B ) {
      
    // Process the message
    process_file_send( proc_data );      
  }
      
  // If it is a File Delete message
  if ( message_type == FILE_DEL_B ) {
      
    // Process the message
    process_file_delete( proc_data );      
  }

END_PROCESS_INCOMING_REQUEST:

  // Cleanup
  if (incoming_message != NULL) {
    free(incoming_message);
  }
  if (recbuf != NULL) {
    free(recbuf);
  }

  {
    int proc_id = proc_data->process_id;

    SOCKET_CLOSE(&(proc_data->connection));
    
    free(processes[proc_id].proc_data);
    processes[proc_id].proc_data = NULL;

    processes[proc_id].is_active = FALSE;
  }

  return;
}
Пример #26
0
void Enqueue::visit(StmtMatmul& s)
{
    if (_failure) return;
    if (_printer) _printer->visit(s);

    // some compute devices are single precision only
    if ( ! OCLhacks::singleton().supportFP64(_memMgr.deviceNum()) &&
         (PrecType::Double == s.precA() ||
          PrecType::Double == s.precB() ||
          PrecType::Double == s.precC()) )
    {
        _failure = true;
        return;
    }

    if (s.isMATMUL())
    {
        // check if device supports the Evergreen matrix multiply
        if (OCLhacks::singleton().supportEvergreen( _memMgr.deviceNum() ))
        {
            // do not allow modulo stream array subscripting for K dimension
            const size_t KfromA = s.isTransposeA() ? s.heightA() : s.widthA();
            const size_t KfromB = s.isTransposeB() ? s.widthB() : s.heightB();
            if (KfromA != KfromB)
            {
                _failure = true;
                return;
            }

            // ATI Evergreen matrix multiply
            Evergreen::MatmulMM matmul( _memMgr.deviceNum() );

            // exogenous parameters
            matmul.setBatching(_vt.numTraces());
            if (s.isSameDataA()) matmul.setSameDataMatrixA();
            if (s.isSameDataB()) matmul.setSameDataMatrixB();
            matmul.setGeneral(s.isGEMM());
            matmul.setPrecision(s.precA(),
                                s.precB(),
                                s.precC());
            matmul.setDimensions(s.heightC(), // M
                                 s.widthC(),  // N
                                 KfromA);     // K
            matmul.setDataLayout(s.isTransposeA(),
                                 s.isTransposeB());

            const vector< size_t > params = _jitMemo.autotuneLookup(matmul);

            if (params.empty())
            {
                _failure = true;
                return;
            }

            matmul.setParams(params);

            ArrayBuf* A
                = s.astvarA()->isTraceVariable()
                      ? _memMgr.arrayBuf(s.astvarA()->variable(), _vt)
                      : _memMgr.arrayBuf(s.astvarA(), _vt);

            ArrayBuf* B
                = s.astvarB()->isTraceVariable()
                      ? _memMgr.arrayBuf(s.astvarB()->variable(), _vt)
                      : _memMgr.arrayBuf(s.astvarB(), _vt);

            ArrayBuf* C
                = s.astvarC()->isTraceVariable()
                      ? _memMgr.arrayBuf(s.astvarC()->variable(), _vt)
                      : _memMgr.arrayBuf(s.astvarC(), _vt);

            const double alpha = s.alpha();
            const double beta  = s.beta();

            const string kernelName = matmul.kernelName();
            stringstream ss;
            ss << matmul;
            const string kernelSource = ss.str();

            OCLkernel ckernel( *_memMgr.computeDevice() );
            ckernel.buildJIT(kernelName,
                             kernelSource);

            if (! ckernel.isOk())
            {
#ifdef __LOGGING_ENABLED__
                stringstream ss;
                ss << "compile error: " << kernelName;
                LOGGER(ss.str())
#endif
                _failure = true;
                return;
            }

            if (! matmul.setArgs(ckernel, A, B, C, alpha, beta))
            {
#ifdef __LOGGING_ENABLED__
                stringstream ss;
                ss << "set arguments error: " << kernelName;
                LOGGER(ss.str())
#endif
                _failure = true;
                return;
            }

            const vector< size_t > globalDims = matmul.globalWorkItems();
            const vector< size_t > localDims = matmul.localWorkItems();

            for (size_t i = 0; i < globalDims.size(); i++)
            {
                ckernel << OCLWorkIndex(globalDims[i], localDims[i]);

                if (! ckernel.statusOp())
                {
#ifdef __LOGGING_ENABLED__
                    stringstream ss;
                    ss << "set index space dimension error: " << i;
                    LOGGER(ss.str())
#endif
                    _failure = true;
                    return;
                }
            }

            *_memMgr.computeDevice() << ckernel;

            if (! _memMgr.computeDevice()->statusOp())
            {
#ifdef __LOGGING_ENABLED__
                stringstream ss;
                ss << "enqueue error: " << kernelName;
                LOGGER(ss.str())
#endif
                _failure = true;
                return;
            }

            *_memMgr.computeDevice() << FLUSH;

            if (! _memMgr.computeDevice()->statusOp())
            {
#ifdef __LOGGING_ENABLED__
                stringstream ss;
                ss << "wait error: " << kernelName;
                LOGGER(ss.str())
#endif
                _failure = true;
                return;
            }

            matmul.clearArgs();
        }
        else
        {
            // there are no other GPU matrix multiply implementations except
            // "Evergreen" (which works on both ATI and NVIDIA), so fail
            _failure = true;
        }
    }
    else if (s.isMATVEC())
    {
        // check if device supports the Evergreen matrix multiply
        if (OCLhacks::singleton().supportEvergreen( _memMgr.deviceNum() ))
        {
            // do not allow modulo stream array subscripting for N dimension
            const size_t NfromA = s.isTransposeA() ? s.heightA() : s.widthA();
            const size_t NfromB = s.widthB();
            if (NfromA != NfromB)
            {
                _failure = true;
                return;
            }

            // ATI Evergreen matrix-vector multiply
            Evergreen::MatmulMV matvec( _memMgr.deviceNum() );

            // exogenous parameters
            matvec.setBatching(_vt.numTraces());
            if (s.isSameDataA()) matvec.setSameDataMatrixA();
            matvec.setGeneral(s.isGEMM());
            matvec.setPrecision(s.precA(),
                                s.precB(),
                                s.precC());
            matvec.setDimensions(s.widthC(), // M
                                 NfromA);    // N
            matvec.setDataLayout(s.isTransposeA());

            const vector< size_t > params = _jitMemo.autotuneLookup(matvec);

            if (params.empty())
            {
                _failure = true;
                return;
            }

            matvec.setParams(params);

            ArrayBuf* A
                = s.astvarA()->isTraceVariable()
                      ? _memMgr.arrayBuf(s.astvarA()->variable(), _vt)
                      : _memMgr.arrayBuf(s.astvarA(), _vt);

            ArrayBuf* B
                = s.astvarB()->isTraceVariable()
                      ? _memMgr.arrayBuf(s.astvarB()->variable(), _vt)
                      : _memMgr.arrayBuf(s.astvarB(), _vt);

            ArrayBuf* C
                = s.astvarC()->isTraceVariable()
                      ? _memMgr.arrayBuf(s.astvarC()->variable(), _vt)
                      : _memMgr.arrayBuf(s.astvarC(), _vt);

            const double alpha = s.alpha();
            const double beta  = s.beta();

            const string kernelName = matvec.kernelName();
            stringstream ss;
            ss << matvec;
            const string kernelSource = ss.str();

            OCLkernel ckernel( *_memMgr.computeDevice() );
            ckernel.buildJIT(kernelName,
                             kernelSource);

            if (! ckernel.isOk())
            {
#ifdef __LOGGING_ENABLED__
                stringstream ss;
                ss << "compile error: " << kernelName;
                LOGGER(ss.str())
#endif
                _failure = true;
                return;
            }

            if (! matvec.setArgs(ckernel, A, B, C, alpha, beta))
            {
#ifdef __LOGGING_ENABLED__
                stringstream ss;
                ss << "set arguments error: " << kernelName;
                LOGGER(ss.str())
#endif
                _failure = true;
                return;
            }

            const vector< size_t > globalDims = matvec.globalWorkItems();
            const vector< size_t > localDims = matvec.localWorkItems();

            for (size_t i = 0; i < globalDims.size(); i++)
            {
                ckernel << OCLWorkIndex(globalDims[i], localDims[i]);

                if (! ckernel.statusOp())
                {
#ifdef __LOGGING_ENABLED__
                    stringstream ss;
                    ss << "set index space dimension error: " << i;
                    LOGGER(ss.str())
#endif
                    _failure = true;
                    return;
                }
            }

            *_memMgr.computeDevice() << ckernel;

            if (! _memMgr.computeDevice()->statusOp())
            {
#ifdef __LOGGING_ENABLED__
                stringstream ss;
                ss << "enqueue error: " << kernelName;
                LOGGER(ss.str())
#endif
                _failure = true;
                return;
            }

            *_memMgr.computeDevice() << FLUSH;

            if (! _memMgr.computeDevice()->statusOp())
            {
#ifdef __LOGGING_ENABLED__
                stringstream ss;
                ss << "wait error: " << kernelName;
                LOGGER(ss.str())
#endif
                _failure = true;
                return;
            }

            matvec.clearArgs();
        }
        else
        {
            // there are no other GPU matrix multiply implementations except
            // "Evergreen" (which works on both ATI and NVIDIA), so fail
            _failure = true;
        }
    }
    else if (s.isVECMAT())
    {
        // check if device supports the Evergreen matrix multiply
        if (OCLhacks::singleton().supportEvergreen( _memMgr.deviceNum() ))
        {
            // do not allow modulo stream array subscripting for N dimension
            const size_t NfromA = s.widthA();
            const size_t NfromB = s.isTransposeB() ? s.widthB() : s.heightB();
            if (NfromA != NfromB)
            {
                _failure = true;
                return;
            }

            // ATI Evergreen matrix-vector multiply
            Evergreen::MatmulMV matvec( _memMgr.deviceNum() );

            // exogenous parameters
            matvec.setBatching(_vt.numTraces());
            if (s.isSameDataB()) matvec.setSameDataMatrixA();
            matvec.setGeneral(s.isGEMM());
            matvec.setPrecision(s.precB(),
                                s.precA(),
                                s.precC());
            matvec.setDimensions(s.widthC(), // M
                                 NfromB);    // N
            matvec.setDataLayout(! s.isTransposeB());

            const vector< size_t > params = _jitMemo.autotuneLookup(matvec);

            if (params.empty())
            {
                _failure = true;
                return;
            }

            matvec.setParams(params);

            ArrayBuf* A
                = s.astvarA()->isTraceVariable()
                      ? _memMgr.arrayBuf(s.astvarA()->variable(), _vt)
                      : _memMgr.arrayBuf(s.astvarA(), _vt);

            ArrayBuf* B
                = s.astvarB()->isTraceVariable()
                      ? _memMgr.arrayBuf(s.astvarB()->variable(), _vt)
                      : _memMgr.arrayBuf(s.astvarB(), _vt);

            ArrayBuf* C
                = s.astvarC()->isTraceVariable()
                      ? _memMgr.arrayBuf(s.astvarC()->variable(), _vt)
                      : _memMgr.arrayBuf(s.astvarC(), _vt);

            const double alpha = s.alpha();
            const double beta  = s.beta();

            const string kernelName = matvec.kernelName();
            stringstream ss;
            ss << matvec;
            const string kernelSource = ss.str();

            OCLkernel ckernel( *_memMgr.computeDevice() );
            ckernel.buildJIT(kernelName,
                             kernelSource);

            if (! ckernel.isOk())
            {
#ifdef __LOGGING_ENABLED__
                stringstream ss;
                ss << "compile error: " << kernelName;
                LOGGER(ss.str())
#endif
                _failure = true;
                return;
            }

            if (! matvec.setArgs(ckernel, B, A, C, alpha, beta))
            {
#ifdef __LOGGING_ENABLED__
                stringstream ss;
                ss << "set arguments error: " << kernelName;
                LOGGER(ss.str())
#endif
                _failure = true;
                return;
            }

            const vector< size_t > globalDims = matvec.globalWorkItems();
            const vector< size_t > localDims = matvec.localWorkItems();

            for (size_t i = 0; i < globalDims.size(); i++)
            {
                ckernel << OCLWorkIndex(globalDims[i], localDims[i]);

                if (! ckernel.statusOp())
                {
#ifdef __LOGGING_ENABLED__
                stringstream ss;
                ss << "set index space dimension error: " << i;
                LOGGER(ss.str())
#endif
                    _failure = true;
                    return;
                }
            }

            *_memMgr.computeDevice() << ckernel;

            if (! _memMgr.computeDevice()->statusOp())
            {
#ifdef __LOGGING_ENABLED__
                stringstream ss;
                ss << "enqueue error: " << kernelName;
                LOGGER(ss.str())
#endif
                _failure = true;
                return;
            }

            *_memMgr.computeDevice() << FLUSH;

            if (! _memMgr.computeDevice()->statusOp())
            {
#ifdef __LOGGING_ENABLED__
                stringstream ss;
                ss << "wait error: " << kernelName;
                LOGGER(ss.str())
#endif
                _failure = true;
                return;
            }

            matvec.clearArgs();
        }
        else
        {
            // there are no other GPU matrix multiply implementations except
            // "Evergreen" (which works on both ATI and NVIDIA), so fail
            _failure = true;
        }
    }
}
Пример #27
0
/**
 * Processes a File Send message from the client, 
 * performs and finalizes the operation
 *
 * @param proc_data_arg           data structure with connection parameters 
 *                                and received message
 */
void process_file_send( PROCESS_DATA_T * proc_data ) {

  const int NOT_FOUND = -1;

  char * filename = NULL;
  char * content  = NULL;

  char * request  = NULL;
  char * response = NULL;

  int search_path_res         = 0;
  int search_length_res       = 0;
  int search_content_res      = 0;

  unsigned long response_len  = 0;

  char destination_dir[2048];
  char l_msg[4096];

  int result = RESULT_UNDEFINED;  
    
  unsigned long content_len = 0;
  int param_path_len = 0;//= (proc_data->received_msg_len - HEADER_LEN - strlen(PARAM_PATH));
 
  // Copies the request to a safe buffer for searching
  request = (char*)malloc(sizeof(char) * proc_data->received_msg_len + 1);
  memcpy(request, proc_data->received_message, proc_data->received_msg_len);
  request[proc_data->received_msg_len] = '\0';

  // Finds parameters positions
  search_path_res         = STR_SEARCH( request, PARAM_PATH, HEADER_LEN);
  search_length_res       = STR_SEARCH( request, PARAM_LENGTH, HEADER_LEN);
  search_content_res      = STR_SEARCH( proc_data->received_message, PARAM_CONTENT, HEADER_LEN);

  if (search_path_res == NOT_FOUND || search_length_res == NOT_FOUND || search_content_res == NOT_FOUND) {
    
    result = RESULT_INVALID_REQUEST;
    goto END_PROCESS_FILE_SEND;
  }

  param_path_len = ( search_length_res - ( HEADER_LEN + strlen(MSG_SEPARATOR) + strlen(PARAM_PATH) ) );

  // Gets name of file to receive
  filename = (char*)malloc(sizeof(char) * param_path_len + 1);  
  memcpy(filename, &request[search_path_res + strlen(PARAM_PATH)], param_path_len);
  filename[param_path_len] = '\0';

  sprintf(l_msg, "A request has been received to receive the file: %s", filename);
  LOGGER(__FUNCTION__, l_msg);
  
  //
  // Gets content length
  //
  content_len = atoi( &( proc_data->received_message[ ( search_length_res + strlen(PARAM_LENGTH) ) ] ) );
  if (content_len == 0) {

    result = RESULT_INVALID_REQUEST;
    goto END_PROCESS_FILE_SEND;
  }

  //
  // Gets content
  //
  content = (char*)malloc(sizeof(char) * content_len + 1);
  memcpy(content, &(proc_data->received_message[ ( search_content_res + strlen(PARAM_CONTENT) ) ]), content_len);
  content[content_len] = '\0';
  
  // Makes a backup copy if file exists
  if (file_exists(filename) == TRUE) {
  
    char new_name[2048];    
    sprintf(new_name, "%s.bkp", filename);
  
    if ( ! file_copy(filename, new_name, TRUE) ) {
    
      sprintf(l_msg, "ERROR: Could not make backup copy of file (%s).", filename);
      LOGGER(__FUNCTION__, l_msg);
    
    }
    
  }
  
  // Prepares directories
  file_get_base_path(filename, destination_dir);
  if (destination_dir != NULL)
  {
    // If destination directory is not root
    if (strcmp(destination_dir, ROOT_DIR) != 0 ) {

      if ( ! file_directory_exists(destination_dir)) {

        if ( ! file_mkdir_parent(destination_dir) ) {

          result = RESULT_COULD_NOT_CREATE_DESTINATION_DIRECTORY;
          goto END_PROCESS_FILE_SEND;
        }
      }
    }
  }
  else {
      
    sprintf(l_msg, "ERROR: The directory specified for the file %s is not valid.", filename);
    LOGGER(__FUNCTION__, l_msg);

    result = RESULT_INVALID_DESTINATION_DIRECTORY;
    goto END_PROCESS_FILE_SEND;
  }

  //
  // Writes base64 to a file, then decodes and unpacks file
  //
  {    
    char b64_filename[2048];
    char gz_filename[2048];
    FILE * f = NULL; 
    
    sprintf(b64_filename, "%s.b64", filename);
    sprintf(gz_filename, "%s.gz", filename);

    f = fopen(b64_filename, "w");
    if (f != NULL) {
      fwrite( content, sizeof(char), content_len, f );
      fclose(f);

      if ( base64_process_file( 'd', b64_filename, gz_filename, content_len ) ) {
    
        if ( gz_unpack_file(gz_filename, filename) ) {
      
          sprintf(l_msg, "file was succesfully unpacked (%s).", gz_filename);
          LOGGER( __FUNCTION__, l_msg );

          result = RESULT_SUCCESS;

        }
        else {

          sprintf(l_msg, "ERROR: could not unpack file (%s).", gz_filename);
          LOGGER( __FUNCTION__, l_msg );

          result = RESULT_FILE_DECOMPRESS_ERROR;
        }      
      
      }
      else {
    
        sprintf(l_msg, "ERROR: could not decode file (%s).", b64_filename);
        LOGGER( __FUNCTION__, l_msg );

        result = RESULT_FILE_DECODE_ERROR;
      }
    
      // Delete generated temporary files
      if ( file_exists(b64_filename) ) {
        remove(b64_filename);
      }
    
      if ( file_exists(gz_filename) ) {
        remove(gz_filename);
      }

    }
    
  }
  
END_PROCESS_FILE_SEND:

  // Generates a response message
  response = message_file_send_response( result, &response_len );  

  // Sends the response
  if ( !process_outgoing_message( proc_data->connection, response, response_len ) ) {
    
    LOGGER(__FUNCTION__, "File Receive operation response message could not be sent.");
  }

  // Cleanup
  if (request != NULL) {
    free(request);
  }
  if (response != NULL) {
    free(response);
  }
  if (filename != NULL) {
    free(filename);
  }
  if (content != NULL) {
    free(content);
  }

  return;
}
Пример #28
0
/// main.
int main(int argc, char** argv) {
    uint32_t maxrow=0, nrpd=0;
    int nparts, ndigits, ierr;

    // must have the output directory name
    if (argc < 2) {
	std::cerr << "\nUsage:\n" << *argv
		  << " <output-dir> [#rows [#rows-per-dir [conf-file]]]\n"
		  << "If the 4th argument is not provided, the number of "
	    "rows per directory will be determined by the memory cache size, "
	    "which is by default 1/2 of the physical memory size.\n"
		  << std::endl;
	return -1;
    }

    //ibis::gVerbose = 8;
    // initialize the file manage with the 5th argument
    ibis::init(argc>4 ? argv[4] : (const char*)0);
    ibis::util::timer mytimer(*argv, 0);
    if (argc > 2) // user specified maxrow
	maxrow = (uint32_t)atof(argv[2]);
    if (maxrow <= 0) {
	double tmp = ibis::fileManager::currentCacheSize();
	maxrow = (uint32_t)
	    ibis::util::compactValue(tmp / 120.0, tmp / 80.0);
	nrpd = maxrow;
    }
    if (maxrow < 10)
	maxrow = 10;
    if (argc > 3) // user specified nrpd
	nrpd = (uint32_t) atof(argv[3]);
    if (nrpd <= 0) {
	double tmp = ibis::fileManager::currentCacheSize();
	nrpd = (uint32_t)
	    ibis::util::compactValue(tmp / 120.0, tmp / 80.0);
    }
    if (nrpd > maxrow) nrpd = maxrow;

    ibis::table::row val;
    std::auto_ptr<ibis::tablex> tab(ibis::tablex::create());
    initColumns(*tab, val);
    ierr = tab->reserveBuffer(nrpd);
    if (ierr > 0 && (unsigned)ierr < nrpd)
	nrpd = ierr;
    LOGGER(1) << *argv << ' ' << argv[1] << ' ' << maxrow << ' ' << nrpd
	      << std::endl;
    nparts = maxrow / nrpd;
    nparts += (maxrow > nparts*nrpd);
    ierr = nparts;
    for (ndigits = 1, ierr >>= 4; ierr > 0; ierr >>= 4, ++ ndigits);
    for (uint32_t irow = 1; irow <= maxrow;) {
	const uint32_t end = irow - 1 + nrpd;
	TDList tdl;
	std::string dir = argv[1];
	if (nparts > 1) { // figure out the directory name
	    const char* str = strrchr(argv[1], FASTBIT_DIRSEP);
	    if (str != 0) {
		if (str[1] == 0) {
		    while (str-1 > argv[1]) {
			if (*(str-1) == FASTBIT_DIRSEP) break;
			else -- str;
		    }
		}
		else {
		    ++ str;
		}
	    }
	    std::ostringstream oss;
	    oss << FASTBIT_DIRSEP << (str ? str : "_") << std::hex
		<< std::setprecision(ndigits) << std::setw(ndigits)
		<< std::setfill('0') << irow / nrpd;
	    dir += oss.str();
	}

	for (; irow <= end; ++ irow) {
	    fillRow(irow, val, tdl);
	    ierr = tab->appendRow(val);
	    LOGGER(ierr != 6)
		<< "Warning -- " << *argv << " failed to append row " << irow
		<< " to the in-memory table, appendRow returned " << ierr;
	    LOGGER(irow % 100000 == 0) << " . " << irow;
	}
	LOGGER(1) << "\n";
	ierr = tab->write(dir.c_str());
	LOGGER(ierr < 0)
	    << "Warning -- " << *argv << " failed to write " << tab->mRows()
	    << " rows to " << dir << ", ibis::tablex::write returned " << ierr;
	writeTDList(tdl, dir.c_str());
	tab->clearData();
	tdl.clear();
    }
    return 0;
} // main
Пример #29
0
/**
 * Sends a synchronous message through a connected node
 * 
 * @param connection            conexion on which the message will be sent
 * @param outgoing_message      outgoing message
 * @param outgoing_message_len  outgoing message length
 *
 * @return                      TRUE if message could be sent, otherwise FALSE
 */ 
int process_outgoing_message( SOCKET_T * connection, char * outgoing_message, int outgoing_message_len ) {

  int bsent = 0;
  int selectval = 0;  
  
  long total_bytes_sent = 0;
  long total_bytes_left = 0;
  long chunk_size = HEADER_LEN;
  
  int send_error = FALSE;
  int timed_out = FALSE;
  int message_send_success = FALSE;    
  unsigned long exec_timeout = 0;
  char * progress;
  
  // Updates moment for next timeout
  exec_timeout = GetTickCount() + gl_timeout;

  // Send message loop
  while ( selectval != S_WRITE && abort_processes == FALSE ) {

    // If operation timed out cancel
    if (GetTickCount() > exec_timeout) {
      timed_out = TRUE;
      goto END_PROCESS_OUTGOING_MESSAGE;
    }

    // Once the header was sent sends the rest of the message...
    while (total_bytes_sent < outgoing_message_len && abort_processes == FALSE) {

      // If operation timed out cancel
      if (GetTickCount() > exec_timeout) {
        timed_out = TRUE;
        goto END_PROCESS_OUTGOING_MESSAGE;
      }

      // ...but first it sends only the header
      while (total_bytes_sent < HEADER_LEN && abort_processes == FALSE) {

        // If operation timed out cancel
        if (GetTickCount() > exec_timeout) {
          timed_out = TRUE;
          goto END_PROCESS_OUTGOING_MESSAGE;
        }

        selectval = SOCKET_SELECT(S_TIMEOUT, connection, S_WRITE);
          
        if ( selectval == S_WRITE ) {

          // Attempts to send the message
          if ( ! SOCKET_SEND(connection, outgoing_message, chunk_size, &bsent) ) {

            // Produces error on fail
            send_error = TRUE;
            break; 
          }

          total_bytes_sent += bsent;
        }

      }

      // If message is only a header then it is complete
      if (total_bytes_sent == outgoing_message_len) {
        break;
      }

      progress = &outgoing_message[total_bytes_sent];

      // Updates bytes left to send count
      total_bytes_left = outgoing_message_len - total_bytes_sent;
      if (total_bytes_left > CHUNK_SIZE) {
        chunk_size = CHUNK_SIZE;
      }
      else {
        chunk_size = total_bytes_left;
      }

      selectval = SOCKET_SELECT(S_TIMEOUT, connection, S_WRITE);
          
      if ( selectval == S_WRITE ) {

        // Attempts to send the message following the header
        if ( ! SOCKET_SEND(connection, progress, chunk_size, &bsent) ) {

          // Produces error on fail
          send_error = TRUE;
          break; 
        }

        total_bytes_sent += bsent;
      }

    }

    // If bytes sent has reached total message size
    if ( total_bytes_sent == outgoing_message_len ) {
        
      message_send_success = TRUE;
      break;
    }

  }
  
END_PROCESS_OUTGOING_MESSAGE:

  // If an error occurred or operation timed out
  if ( send_error == TRUE || timed_out) {
  
    char log[_BUFFER_SIZE_L];
    memset(log, 0x00, _BUFFER_SIZE_L);

    strcpy( log, "Failed while attempting to send the following message: ");
    if (outgoing_message_len >= _BUFFER_SIZE_L) {
      memcpy(log, outgoing_message, _BUFFER_SIZE_L - 1);
    }
    else {
      memcpy(log, outgoing_message, outgoing_message_len);
    }
    LOGGER(__FUNCTION__, log);
    
    sprintf( log, "[Number of bytes sent:%ld]", total_bytes_sent );
    LOGGER(__FUNCTION__, log);
  }

  return message_send_success;
    
}
Пример #30
0
 void ProgressLogger::printToConsole(std::string text)
 {
     LOGGER(text);
 }