// =-=-=-=-=-=-=-
// local function which makes the call to chmod via the resource plugin
int _rsFileChmod(
rsComm_t* _comm,
fileChmodInp_t* _chmod_inp ) {
// =-=-=-=-=-=-=-
// make the call to chmod via the resource plugin
irods::file_object_ptr file_obj(
new irods::file_object(
_comm,
_chmod_inp->objPath,
_chmod_inp->fileName,
_chmod_inp->rescHier,
0, 0, 0 ) );
irods::error chmod_err = fileChmod( _comm, file_obj, _chmod_inp->mode );
// =-=-=-=-=-=-=-
// report errors if any
if ( !chmod_err.ok() ) {
std::stringstream msg;
msg << "fileChmod failed for [";
msg << _chmod_inp->fileName;
msg << "] to mode [";
msg << _chmod_inp->mode;
msg << "";
irods::error err = PASSMSG( msg.str(), chmod_err );
irods::log( err );
}
return chmod_err.code();
} //_rsChmod
int _call_file_modified_for_replica(
rsComm_t* rsComm,
regReplica_t* regReplicaInp ) {
int status = 0;
dataObjInfo_t* destDataObjInfo = regReplicaInp->destDataObjInfo;
irods::file_object_ptr file_obj(
new irods::file_object(
rsComm,
destDataObjInfo ) );
char* pdmo_kw = getValByKey( ®ReplicaInp->condInput, IN_PDMO_KW );
if ( pdmo_kw != NULL ) {
file_obj->in_pdmo( pdmo_kw );
}
irods::error ret = fileModified( rsComm, file_obj );
if ( !ret.ok() ) {
std::stringstream msg;
msg << __FUNCTION__;
msg << " - Failed to signal resource that the data object \"";
msg << destDataObjInfo->objPath;
msg << "\" was registered";
ret = PASSMSG( msg.str(), ret );
irods::log( ret );
status = ret.code();
}
return status;
} // _call_file_modified_for_replica
// =-=-=-=-=-=-=-
// local function which makes the call to truncate via the resource plugin
int _rsFileTruncate(
rsComm_t* _comm,
fileOpenInp_t* _trunc_inp ) {
// =-=-=-=-=-=-=-
// make the call to rename via the resource plugin
irods::file_object_ptr file_obj(
new irods::file_object(
_comm,
_trunc_inp->objPath,
_trunc_inp->fileName,
_trunc_inp->resc_hier_,
0, 0, 0 ) );
file_obj->size( _trunc_inp->dataSize );
// =-=-=-=-=-=-=-
// pass condInput
file_obj->cond_input( _trunc_inp->condInput );
irods::error trunc_err = fileTruncate( _comm, file_obj );
// =-=-=-=-=-=-=-
// report errors if any
if ( !trunc_err.ok() ) {
std::stringstream msg;
msg << "fileTruncate for [";
msg << _trunc_inp->fileName;
msg << "]";
msg << trunc_err.code();
irods::error err = PASSMSG( msg.str(), trunc_err );
irods::log( err );
}
return trunc_err.code();
} // _rsFileTruncate
// =-=-=-=-=-=-=-
// local function which makes the call to rename via the resource plugin
int _rsFileRename(
rsComm_t* _comm,
fileRenameInp_t* _rename_inp,
fileRenameOut_t** _rename_out,
rodsServerHost_t* _server_host ) {
// =-=-=-=-=-=-=-
// FIXME: need to check resource permission and vault permission
// when RCAT is available
// mkDirForFilePath( _comm, "/", _rename_inp->newFileName, getDefDirMode () ); - The actual file path depends on the resource
// =-=-=-=-=-=-=-
// make the call to rename via the resource plugin
irods::file_object_ptr file_obj(
new irods::file_object(
_comm,
_rename_inp->objPath,
_rename_inp->oldFileName,
_rename_inp->rescHier,
0, 0, 0 ) );
irods::error rename_err = fileRename( _comm, file_obj, _rename_inp->newFileName );
// =-=-=-=-=-=-=-
// report errors if any
if ( !rename_err.ok() ) {
std::stringstream msg;
msg << "fileRename failed for [";
msg << _rename_inp->oldFileName;
msg << "] to [";
msg << _rename_inp->newFileName;
msg << "]";
irods::error err = PASSMSG( msg.str(), rename_err );
irods::log( err );
}
// =-=-=-=-=-=-=-
// compare fco phy path and old file name
// if they differ then repave for next call
if ( file_obj->physical_path() == _rename_inp->oldFileName ) {
file_obj->physical_path( _rename_inp->newFileName );
}
// =-=-=-=-=-=-=-
// percolate possible change in phy path up
( *_rename_out ) = ( fileRenameOut_t* ) malloc( sizeof( fileRenameOut_t ) );
strncpy(
( *_rename_out )->file_name,
file_obj->physical_path().c_str(),
MAX_NAME_LEN );
return rename_err.code();
} // _rsFileRename
// =-=-=-=-=-=-=-
// local function to handle call to stat via resource plugin
int _rsFileLseek(
rsComm_t* _comm,
fileLseekInp_t* _lseek_inp,
fileLseekOut_t** _lseek_out ) {
// =-=-=-=-=-=-=-
// make call to lseek via resource plugin
irods::file_object_ptr file_obj(
new irods::file_object(
_comm,
FileDesc[_lseek_inp->fileInx].objPath,
FileDesc[_lseek_inp->fileInx].fileName,
FileDesc[_lseek_inp->fileInx].rescHier,
FileDesc[_lseek_inp->fileInx].fd,
0, 0 ) );
irods::error lseek_err = fileLseek(
_comm,
file_obj,
_lseek_inp->offset,
_lseek_inp->whence );
// =-=-=-=-=-=-=-
// handle error conditions and log
if ( !lseek_err.ok() ) {
std::stringstream msg;
msg << "lseek failed for [";
msg << FileDesc[_lseek_inp->fileInx].fileName;
msg << "]";
irods::error ret_err = PASSMSG( msg.str(), lseek_err );
irods::log( ret_err );
return lseek_err.code();
}
else {
( *_lseek_out ) = ( fileLseekOut_t* )malloc( sizeof( fileLseekOut_t ) );
memset( ( *_lseek_out ), 0, sizeof( fileLseekOut_t ) );
( *_lseek_out )->offset = lseek_err.code();
return 0;
}
} // _rsFileLseek
int _rsFileRead(
rsComm_t* _comm,
fileReadInp_t* _read_inp,
bytesBuf_t* _read_bbuf ) {
// =-=-=-=-=-=-=-
// XXXX need to check resource permission and vault permission
// when RCAT is available
// =-=-=-=-=-=-=-
// call resource plugin for POSIX read
irods::file_object_ptr file_obj(
new irods::file_object(
_comm,
FileDesc[_read_inp->fileInx].objPath,
FileDesc[_read_inp->fileInx].fileName,
FileDesc[_read_inp->fileInx].rescHier,
FileDesc[_read_inp->fileInx].fd,
0, 0 ) );
irods::error ret = fileRead( _comm,
file_obj,
_read_bbuf->buf,
_read_inp->len );
// =-=-=-=-=-=-=
// log an error if the read failed,
// pass long read error
if ( !ret.ok() ) {
std::stringstream msg;
msg << "fileRead failed for ";
msg << file_obj->physical_path();
msg << "]";
irods::error err = PASSMSG( msg.str(), ret );
irods::log( err );
}
else {
_read_bbuf->len = ret.code();
}
return ret.code();
} // _rsFileRead
// =-=-=-=-=-=-=-
// local function to handle call to stat via resource plugin
int _rsFileStat(
rsComm_t* _comm,
fileStatInp_t* _stat_inp,
rodsStat_t** _stat_out ) {
struct stat myFileStat;
memset( &myFileStat, 0, sizeof( myFileStat ) );
// =-=-=-=-=-=-=-
// make call to stat via resource plugin
irods::file_object_ptr file_obj(
new irods::file_object(
_comm,
_stat_inp->objPath,
_stat_inp->fileName,
_stat_inp->rescId,
0, 0, 0 ) );
irods::error stat_err = fileStat( _comm, file_obj, &myFileStat );
// =-=-=-=-=-=-=-
// log error if necessary
if ( !stat_err.ok() ) {
// irods::log(LOG_ERROR, stat_err.result());
return stat_err.code();
}
// =-=-=-=-=-=-=-
// convert unix stat struct to an irods stat struct
*_stat_out = ( rodsStat_t* )malloc( sizeof( rodsStat_t ) );
int status = statToRodsStat( *_stat_out, &myFileStat );
// =-=-=-=-=-=-=-
// manage error if necessary
if ( status < 0 ) {
free( *_stat_out );
*_stat_out = NULL;
}
return status;
} // _rsFileStat
int
_rsRegDataObj( rsComm_t *rsComm, dataObjInfo_t *dataObjInfo ) {
#ifdef RODS_CAT
int status;
irods::error ret;
status = chlRegDataObj( rsComm, dataObjInfo );
if ( status < 0 ) {
char* sys_error;
char* rods_error = rodsErrorName( status, &sys_error );
std::stringstream msg;
msg << __FUNCTION__;
msg << " - Failed to register data object \"" << dataObjInfo->objPath << "\"";
msg << " - " << rods_error << " " << sys_error;
ret = ERROR( status, msg.str() );
irods::log( ret );
}
else {
irods::file_object_ptr file_obj(
new irods::file_object(
rsComm,
dataObjInfo ) );
ret = fileRegistered( rsComm, file_obj );
if ( !ret.ok() ) {
std::stringstream msg;
msg << __FUNCTION__;
msg << " - Failed to signal resource that the data object \"";
msg << dataObjInfo->objPath;
msg << "\" was registered";
ret = PASSMSG( msg.str(), ret );
irods::log( ret );
status = ret.code();
}
}
return status;
#else
return SYS_NO_RCAT_SERVER_ERR;
#endif
}
int
_l3DataPutSingleBuf( rsComm_t *rsComm, int l1descInx, dataObjInp_t *dataObjInp,
bytesBuf_t *dataObjInpBBuf ) {
int status = 0;
dataObjInfo_t *myDataObjInfo = L1desc[l1descInx].dataObjInfo;
int bytesWritten = l3FilePutSingleBuf( rsComm, l1descInx, dataObjInpBBuf );
if ( bytesWritten >= 0 ) {
if ( L1desc[l1descInx].replStatus == NEWLY_CREATED_COPY &&
myDataObjInfo->specColl == NULL &&
L1desc[l1descInx].remoteZoneHost == NULL ) {
/* the check for remoteZoneHost host is not needed because
* the put would have done in the remote zone. But it make
* the code easier to read (similar ro copy).
*/
status = svrRegDataObj( rsComm, myDataObjInfo );
if ( status < 0 ) {
rodsLog( LOG_NOTICE,
"l3DataPutSingleBuf: rsRegDataObj for %s failed, status = %d",
myDataObjInfo->objPath, status );
if ( status != CATALOG_ALREADY_HAS_ITEM_BY_THAT_NAME ) {
l3Unlink( rsComm, myDataObjInfo );
}
return status;
}
else {
myDataObjInfo->replNum = status;
}
}
/* myDataObjInfo->dataSize = bytesWritten; update size problem */
if ( bytesWritten == 0 && myDataObjInfo->dataSize > 0 ) {
/* overwrite with 0 len file */
L1desc[l1descInx].bytesWritten = 1;
}
else {
L1desc[l1descInx].bytesWritten = bytesWritten;
}
// special case of zero length files, trigger the fileModified
// operation for execution of coordinating resource logic
if ( 0 == dataObjInp->dataSize ) {
irods::file_object_ptr file_obj(
new irods::file_object(
rsComm,
myDataObjInfo ) );
char* pdmo_kw = getValByKey( &myDataObjInfo->condInput, IN_PDMO_KW );
if ( pdmo_kw != NULL ) {
file_obj->in_pdmo( pdmo_kw );
}
irods::error ret = fileModified( rsComm, file_obj );
if ( !ret.ok() ) {
std::stringstream msg;
msg << "fileModified failed for [";
msg << myDataObjInfo->objPath;
msg << "]";
ret = PASSMSG( msg.str(), ret );
irods::log( ret );
status = ret.code();
}
}
}
L1desc[l1descInx].dataSize = dataObjInp->dataSize;
return bytesWritten;
}
int
rsDataObjOpen( rsComm_t *rsComm, dataObjInp_t *dataObjInp ) {
int status, l1descInx;
int remoteFlag;
rodsServerHost_t *rodsServerHost;
remoteFlag = getAndConnRemoteZone( rsComm, dataObjInp, &rodsServerHost,
REMOTE_OPEN );
if ( remoteFlag < 0 ) {
return remoteFlag;
}
else if ( remoteFlag == REMOTE_HOST ) {
openStat_t *openStat = NULL;
status = rcDataObjOpenAndStat( rodsServerHost->conn, dataObjInp,
&openStat );
if ( status < 0 ) {
return status;
}
l1descInx = allocAndSetL1descForZoneOpr( status, dataObjInp,
rodsServerHost, openStat );
if ( openStat != NULL ) {
free( openStat );
}
return l1descInx;
}
else {
// dataObjInfo_t linked list
dataObjInfo_t *dataObjInfoHead = NULL;
// resource hierarchy
std::string hier;
// =-=-=-=-=-=-=-
// determine the resource hierarchy if one is not provided
if ( getValByKey( &dataObjInp->condInput, RESC_HIER_STR_KW ) == NULL ) {
irods::error ret = irods::resolve_resource_hierarchy(
irods::OPEN_OPERATION,
rsComm,
dataObjInp,
hier,
&dataObjInfoHead );
if (ret.ok()) {
// =-=-=-=-=-=-=-
// we resolved the redirect and have a host, set the hier str for subsequent
// api calls, etc.
addKeyVal( &dataObjInp->condInput, RESC_HIER_STR_KW, hier.c_str() );
}
}
else {
// file object for file_object_factory
irods::file_object_ptr file_obj( new irods::file_object() );
// get resource hierarchy from condInput
hier = getValByKey( &dataObjInp->condInput, RESC_HIER_STR_KW );
// get replicas vector populated
irods::error fac_err = irods::file_object_factory(rsComm, dataObjInp, file_obj, &dataObjInfoHead);
}// if ( getValByKey( &dataObjInp->condInput, RESC_HIER_STR_KW ) == NULL )
l1descInx = _rsDataObjOpen( rsComm, dataObjInp, dataObjInfoHead );
}
return l1descInx;
}
int main(int argc, char *argv[])
{
QCoreApplication a(argc, argv);
QString file_path = "/home/eveil/src/dev/qt-json/Load-JSON-deep/test.json";
QFile file_obj(file_path);
if(!file_obj.open(QIODevice::ReadOnly)){
qDebug()<< "Failed to open file: " << file_path;
exit(1);
}
QTextStream file_text(&file_obj);
QString json_string;
json_string = file_text.readAll();
file_obj.close();
QByteArray json_bytes = json_string.toLocal8Bit();
auto json_doc = QJsonDocument::fromJson(json_bytes);
if(json_doc.isNull()){
qDebug()<< "Failed to creat json doc from: "
<<endl<<json_bytes;
exit(2);
}
qDebug() << json_doc;
if(!json_doc.isObject()){
qDebug() << "json doc is not object.";
exit(3);
}
QJsonObject root_obj=json_doc.object();
QVariantMap root_map=root_obj.toVariantMap();
QVariantMap stats=root_map["stats"].toMap();
QVariantList inv=root_map["inventory"].toList();
if(stats.isEmpty()){
qDebug() << "The stats object is empty.";
exit(4);
}
if(inv.isEmpty()){
qDebug() << "The inventory array is empty";
}
// iterate a json object's members:
QStringList key_list = stats.keys();
for(int i=0; i < key_list.count(); ++i){
QString key=key_list.at(i);
int stat_val = stats[key.toLocal8Bit()].toInt();
qDebug() << key << ": " << stat_val;
}
qDebug() << "Inventory:";
for(int i=0; i < inv.count(); ++i){
qDebug() << inv.at(i).toString();
}
return a.exec();
}
// =-=-=-=-=-=-=-=
// _rsFileSyncToArch - this the local version of rsFileSyncToArch.
int _rsFileSyncToArch(
rsComm_t* _comm,
fileStageSyncInp_t* _sync_inp,
fileSyncOut_t** _sync_out ) {
// =-=-=-=-=-=-=-
// XXXX need to check resource permission and vault permission
// when RCAT is available
// =-=-=-=-=-=-=-
// prep
if ( _sync_inp->objPath[0] == '\0' ) {
std::stringstream msg;
msg << __FUNCTION__;
msg << " - Empty logical path.";
irods::log( LOG_ERROR, msg.str() );
return SYS_INVALID_INPUT_PARAM;
}
// =-=-=-=-=-=-=-
// make call to synctoarch via resource plugin
irods::file_object_ptr file_obj(
new irods::file_object(
_comm,
_sync_inp->objPath,
_sync_inp->filename, "", 0,
_sync_inp->mode,
_sync_inp->flags ) );
file_obj->resc_hier( _sync_inp->rescHier );
// =-=-=-=-=-=-=-
// pass condInput
file_obj->cond_input( _sync_inp->condInput );
irods::error sync_err = fileSyncToArch( _comm, file_obj, _sync_inp->cacheFilename );
if ( !sync_err.ok() ) {
if ( getErrno( sync_err.code() ) == ENOENT ) {
// =-=-=-=-=-=-=-
// the directory does not exist, lets make one
int status = mkDirForFilePath(
_comm,
0,
_sync_inp->filename,
_sync_inp->rescHier,
getDefDirMode() );
if ( status < 0 ) {
rodsLog( LOG_ERROR, "mkDirForFilePath failed in _rsFileSyncToArch with status %d", status );
return status;
}
}
else if ( getErrno( sync_err.code() ) == EEXIST ) {
// =-=-=-=-=-=-=-
// an empty dir may be there, make the call to rmdir via the resource plugin
irods::collection_object_ptr coll_obj(
new irods::collection_object(
_sync_inp->filename,
_sync_inp->rescHier,
0, 0 ) );
coll_obj->cond_input( _sync_inp->condInput );
irods::error rmdir_err = fileRmdir( _comm, coll_obj );
if ( !rmdir_err.ok() ) {
std::stringstream msg;
msg << "fileRmdir failed for [";
msg << _sync_inp->filename;
msg << "]";
irods::error err = PASSMSG( msg.str(), sync_err );
irods::log( err );
}
}
else {
std::stringstream msg;
msg << "fileSyncToArch failed for [";
msg << _sync_inp->filename;
msg << "]";
irods::error err = PASSMSG( msg.str(), sync_err );
irods::log( err );
return sync_err.code();
}
// =-=-=-=-=-=-=-
// make call to synctoarch via resource plugin
sync_err = fileSyncToArch( _comm, file_obj, _sync_inp->cacheFilename );
if ( !sync_err.ok() ) {
std::stringstream msg;
msg << "fileSyncToArch failed for [";
msg << _sync_inp->filename;
msg << "]";
msg << sync_err.code();
irods::error err = PASSMSG( msg.str(), sync_err );
irods::log( err );
}
} // if !sync_err.ok()
// =-=-=-=-=-=-=-
// has the file name has changed?
if ( *_sync_out ) {
rstrcpy( ( *_sync_out )->file_name, file_obj->physical_path().c_str(), MAX_NAME_LEN );
}
return sync_err.code();
} // _rsFileSyncToArch
int _rsFileGet(
rsComm_t* _comm,
fileOpenInp_t* _get_inp,
bytesBuf_t* _get_buf ) {
int fd;
int len;
len = _get_inp->dataSize;
fd = _rsFileOpen( _comm, _get_inp );
if ( fd < 0 ) {
rodsLog( LOG_NOTICE,
"_rsFileGet: fileGet for %s, status = %d",
_get_inp->fileName, fd );
return fd;
}
if ( _get_buf->buf == NULL ) {
_get_buf->buf = malloc( len );
}
irods::file_object_ptr file_obj(
new irods::file_object(
_comm,
_get_inp->objPath,
_get_inp->fileName,
_get_inp->resc_hier_,
fd,
_get_inp->mode,
_get_inp->flags ) );
// =-=-=-=-=-=-=-
// pass condInput
file_obj->cond_input( _get_inp->condInput );
irods::error read_err = fileRead( _comm,
file_obj,
_get_buf->buf,
len );
int bytes_read = read_err.code();
if ( bytes_read != len ) {
if ( bytes_read >= 0 ) {
_get_buf->len = bytes_read;
}
else {
std::stringstream msg;
msg << "fileRead failed for [";
msg << _get_inp->fileName;
msg << "]";
irods::error ret_err = PASSMSG( msg.str(), read_err );
irods::log( ret_err );
}
}
else {
_get_buf->len = bytes_read;
}
// =-=-=-=-=-=-=-
// call resource plugin close
irods::error close_err = fileClose( _comm,
file_obj );
if ( !close_err.ok() ) {
irods::error err = PASSMSG( "error on close", close_err );
irods::log( err );
}
return bytes_read;
} // _rsFileGet
std::string
Java_Compiler_Dump_Algorithm::process ( boost::uint32_t compiler_id,
std::string filename )
{
#ifdef LIBREVERSE_DEBUG
Trace::write_Trace ( TraceArea::GRNN_OPTIMIZER,
TraceLevel::DETAIL,
"Entering Java_Compiler_Dump_Algorithm::process" );
#endif /* LIBREVERSE_DEBUG */
std::stringstream output;
try
{
reverse::io_types::File_ID::ptr_t file_obj ( new reverse::io::File_ID ( filename ) );
reverse::java_module::Reader r_obj ( file_obj );
r_obj.read_Class_Header();
reverse::data_types::memory_map::ptr_t mem_ptr = r_obj.get_memory_map();
output << boost::format(" <%1%>") % classifier::Java_Input_Tag_Names::TAG_FILE << std::endl;
output << boost::format ( " <%1%>%2%</%3%>" )
% classifier::Java_Input_Tag_Names::TAG_TARGET_ID
% compiler_id
% classifier::Java_Input_Tag_Names::TAG_TARGET_ID << std::endl;
output << boost::format ( " <%1%>%2%</%3%>" )
% classifier::Java_Input_Tag_Names::TAG_FILESIZE
% ( mem_ptr->size() )
% classifier::Java_Input_Tag_Names::TAG_FILESIZE << std::endl;
// Get class header
reverse::java_types::Class_Header::ptr_t hdr_ptr = r_obj.get_Header();
// Grab data
std::vector<float> stats(12,0);
collect_Constant_Pool_Stats ( stats, hdr_ptr->get_Constant_Pool_Begin(), hdr_ptr->get_Constant_Pool_End() );
// Add entry to output file
output << boost::format ( " <%1%>%2%.%3%</%4%>" )
% classifier::Java_Input_Tag_Names::TAG_VERSION
% hdr_ptr->get_Major_Version()
% hdr_ptr->get_Minor_Version()
% classifier::Java_Input_Tag_Names::TAG_VERSION << std::endl;
output << boost::format ( " <%1%>%2%</%3%>" )
% classifier::Java_Input_Tag_Names::TAG_THIS_INDEX
% hdr_ptr->get_This_Class()
% classifier::Java_Input_Tag_Names::TAG_THIS_INDEX << std::endl;
output << boost::format ( " <%1%>%2%</%3%>" )
% classifier::Java_Input_Tag_Names::TAG_SUPER_INDEX
% hdr_ptr->get_Super_Class()
% classifier::Java_Input_Tag_Names::TAG_SUPER_INDEX << std::endl;
output << boost::format ( " <%1%>%2%</%3%>" )
% classifier::Java_Input_Tag_Names::TAG_CONSTANT_POOL_COUNT
% hdr_ptr->get_Constant_Pool_Count()
% classifier::Java_Input_Tag_Names::TAG_CONSTANT_POOL_COUNT << std::endl;
float constant_pool_total = hdr_ptr->get_Constant_Pool_Count();
print_Constant_Pool_Stats ( output, stats, constant_pool_total );
output << boost::format ( " <%1%>%2%</%3%>" )
% classifier::Java_Input_Tag_Names::TAG_FIELD_COUNT
% hdr_ptr->get_Field_Count()
% classifier::Java_Input_Tag_Names::TAG_FIELD_COUNT << std::endl;
output << boost::format ( " <%1%>%2%</%3%>" )
% classifier::Java_Input_Tag_Names::TAG_METHOD_COUNT
% hdr_ptr->get_Method_Count()
% classifier::Java_Input_Tag_Names::TAG_METHOD_COUNT << std::endl;
output << boost::format(" </%1%>" ) % classifier::Java_Input_Tag_Names::TAG_FILE << std::endl;
}
catch ( std::exception &e )
{
#ifdef LIBREVERSE_DEBUG
Trace::write_Trace ( TraceArea::GRNN_OPTIMIZER,
TraceLevel::ERROR,
boost::str ( boost::format("(ERROR) Cannot read %1%. Skipping the file. ") % filename ) );
Trace::write_Trace ( TraceArea::GRNN_OPTIMIZER,
TraceLevel::ERROR,
e.what() );
#endif /* LIBREVERSE_DEBUG */
}
#ifdef LIBREVERSE_DEBUG
Trace::write_Trace ( TraceArea::GRNN_OPTIMIZER,
TraceLevel::DETAIL,
"Exiting Java_Compiler_Dump_Algorithm::process" );
#endif /* LIBREVERSE_DEBUG */
return output.str();
}
int main()
{
time_t start_time = time(NULL);
std::string data_path("D:/Zmisc/Github/NLP/Naive-Bayes-Document-Classifier-master/trial/reuters_modified/"); //change this path according to your local data path
std::ifstream file_cate((data_path+std::string("cats.txt")).c_str());
//if file can't be opened, we exit
if(!file_cate.good())
{
std::cerr << "Can't open cats.txt to read.\n";
//exit(1); which is the same as
return EXIT_FAILURE;
}
std::string line;
std::string token;
std::string word1;
std::string word2;
std::string file_name;
std::string min_cat;
std::string train("training");
std::string tes("test");
//using the tree map implementation with O(log n) time for insertion, search, deletion etc
//because there are no hash table implementations in c++ stl
std::map<std::string,std::string > test_map; //test file to category map
std::map<std::string,std::string > train_map; //train file to category map
std::map<std::string,std::string >::iterator fi_itr;
std::map<std::string,std::string >::const_iterator cfi_itr;
std::map<std::string,int > cate_num_docs_map;
std::map<std::string,int > cate_num_docs_test_map;
std::map<std::string,int >::iterator itr;
std::map<std::string,int >::const_iterator citr;
std::set<std::string> words_set;
std::set<std::string>::iterator set_itr;
std::set<std::string>::const_iterator cset_itr;
std::map<std::string, std::map<std::string,unsigned int> > word_cates_num_map;
std::map<std::string, std::map<std::string,unsigned int> >::iterator big_itr;
std::map<std::string, std::map<std::string,unsigned int> >::const_iterator cbig_itr;
std::map<std::string,unsigned int>::iterator small_itr;
std::map<std::string,unsigned int>::const_iterator csmall_itr;
//Create a map with a word as the key and a map as the value
// in the inner map the category as key and number of documents in that category where it occurs as value
std::map<std::string, std::vector<unsigned int> > cates_results;
std::map<std::string, std::vector<unsigned int> >::iterator res_itr;
std::map<std::string, std::vector<unsigned int> >::const_iterator cres_itr;
unsigned int len_train;
unsigned int counter;
unsigned int error_counter;
unsigned int len_test;
int nct;
double time_diff;
double minimum_neg_log_prob;
double neg_log_prob;
double ratio;
//Format of each line of categs file - filename cate1 cate 2(if any) cate3(if any)
while(getline(file_cate,line))
{
counter = 0;
std::istringstream iss(line);
while(iss >> token)
{
++counter;
if(counter==1)
word1 = token;
else if(counter==2)
word2 = token;
}
if(counter>2) //we only consider files which are in 1 category
continue;
if(word1.compare(0,4,tes)==0)
{
file_name = word1.substr(5);
test_map[file_name] = word2;
++cate_num_docs_test_map[word2];
}
else if(word1.compare(0,8,train)==0)
{
file_name = word1.substr(9);
train_map[file_name] = word2;
++cate_num_docs_map[word2];
}
}
file_cate.close();
//Removing categories with less than 21 files in either the training set or test set
prune_rare_categories(cate_num_docs_map,cate_num_docs_test_map);
for(cfi_itr = train_map.begin();cfi_itr!=train_map.end();++cfi_itr)
{
itr = cate_num_docs_map.find(cfi_itr->second);
//File belongs to a low category
if(itr==cate_num_docs_map.end())
continue;
file_name = data_path+std::string("training/")+cfi_itr->first;
std::ifstream file_obj(file_name.c_str());
//if file can't be opened, we exit
if(!file_obj.good())
{
std::cerr << "Can't open training file to read.\n";
//exit(1); which is the same as
return EXIT_FAILURE;
}
words_set.clear();
while(file_obj >> token)
words_set.insert(token);
for(cset_itr = words_set.begin();cset_itr!=words_set.end();++cset_itr)
{
big_itr = word_cates_num_map.find(*cset_itr);
++word_cates_num_map[*cset_itr][cfi_itr->second];
}
file_obj.close();
++len_train;
}
time_diff = difftime(time(NULL),start_time);
std::cout << "The Classifier is trained and it took "<< time_diff << " seconds.\n" ;
start_time = time(NULL);
//Initialize cates_results
for(citr=cate_num_docs_map.begin();citr!=cate_num_docs_map.end();++citr)
{
cates_results[citr->first].push_back(0);
cates_results[citr->first].push_back(0);
cates_results[citr->first].push_back(0);
cates_results[citr->first].push_back(0);
}
for(cfi_itr = test_map.begin();cfi_itr!=test_map.end();++cfi_itr)
{
itr = cate_num_docs_map.find(cfi_itr->second);
//File belongs to a low category
if(itr==cate_num_docs_map.end())
continue;
file_name = data_path+std::string("test/")+cfi_itr->first;
std::ifstream file_obj(file_name.c_str());
//if file can't be opened, we exit
if(!file_obj.good())
{
std::cerr << "Can't open test file to read.\n";
//exit(1); which is the same as
return EXIT_FAILURE;
}
words_set.clear();
while(file_obj >> token)
words_set.insert(token);
minimum_neg_log_prob=1000000000;
min_cat = "";
for(citr=cate_num_docs_map.begin();citr!=cate_num_docs_map.end();++citr)
{
neg_log_prob = -log(citr->second/(len_train*1.0));
for(cbig_itr = word_cates_num_map.begin(); cbig_itr!=word_cates_num_map.end();++cbig_itr)
{
set_itr = words_set.find(cbig_itr->first);
nct = word_cates_num_map[cbig_itr->first][citr->first];
ratio = (nct+1)/(citr->second+2.0);
if(set_itr!=words_set.end())
neg_log_prob-= log(ratio);
else
neg_log_prob-=log(1-ratio);
}
if(minimum_neg_log_prob>neg_log_prob)
{
min_cat = citr->first;
minimum_neg_log_prob = neg_log_prob;
}
}
file_obj.close();
++len_test;
if(min_cat!=cfi_itr->second)
++error_counter;
for(cres_itr = cates_results.begin();cres_itr!=cates_results.end();++cres_itr)
{
word2 = cres_itr->first;
if(word2==min_cat)
{
if(word2==cfi_itr->second)
++cates_results[word2][0];
else
++cates_results[word2][1];
}
else
{
if(word2==cfi_itr->second)
++cates_results[word2][2];
else
++cates_results[word2][3];
}
}
}
std::cout << "The fraction of errors is " << error_counter/(len_test*1.0) << "\n";
//Evaluation by finer measures
f_measure(cates_results);
time_diff = difftime(time(NULL),start_time);
std::cout << "The Classifier has run and it took "<< time_diff << " seconds.\n" ;
return 0;
}
// =-=-=-=-=-=-=-
// local implementation of put
int _rsFilePut(
rsComm_t* _comm,
fileOpenInp_t* _put_inp,
bytesBuf_t* _put_bbuf,
rodsServerHost_t* _server_host ) {
int fd = 0;
// =-=-=-=-=-=-=-
// NOTE:: this test does not seem to work for i86 solaris
if ( ( _put_inp->otherFlags & FORCE_FLAG ) != 0 ) {
// =-=-=-=-=-=-=-
// create one if it does not exist */
_put_inp->flags |= O_CREAT;
fd = _rsFileOpen( _comm, _put_inp );
}
else {
fd = _rsFileCreate( _comm, _put_inp, _server_host );
} // else
// =-=-=-=-=-=-=-
// log, error if any
if ( fd < 0 ) {
if ( getErrno( fd ) == EEXIST ) {
rodsLog( LOG_DEBUG1,
"_rsFilePut: filePut for %s, status = %d",
_put_inp->fileName, fd );
}
else if ( fd != DIRECT_ARCHIVE_ACCESS ) {
rodsLog( LOG_DEBUG,
"_rsFilePut: filePut for %s, status = %d",
_put_inp->fileName, fd );
}
return ( fd );
}
// =-=-=-=-=-=-=-
// call write for resource plugin
irods::file_object_ptr file_obj(
new irods::file_object(
_comm,
_put_inp->objPath,
_put_inp->fileName,
_put_inp->resc_hier_,
fd, 0, 0 ) );
file_obj->in_pdmo( _put_inp->in_pdmo );
file_obj->cond_input( _put_inp->condInput );
irods::error write_err = fileWrite( _comm,
file_obj,
_put_bbuf->buf,
_put_bbuf->len );
int write_code = write_err.code();
// =-=-=-=-=-=-=-
// log errors, if any
if ( write_code != _put_bbuf->len ) {
if ( write_code >= 0 ) {
std::stringstream msg;
msg << "fileWrite failed for [";
msg << _put_inp->fileName;
msg << "] towrite [";
msg << _put_bbuf->len;
msg << "] written [";
msg << write_code << "]";
irods::error err = PASSMSG( msg.str(), write_err );
irods::log( err );
write_code = SYS_COPY_LEN_ERR;
}
else {
std::stringstream msg;
msg << "fileWrite failed for [";
msg << _put_inp->fileName;
msg << "]";
irods::error err = PASSMSG( msg.str(), write_err );
irods::log( err );
}
}
// =-=-=-=-=-=-=-
// close up after ourselves
irods::error close_err = fileClose( _comm,
file_obj );
if ( !close_err.ok() ) {
irods::error err = PASSMSG( "error on close", close_err );
irods::log( err );
}
// =-=-=-=-=-=-=-
// return 'write_err code' as this includes this implementation
// assumes we are returning the size of the file 'put' via fileWrite
return write_code;
} // _rsFilePut
int
_rsChkNVPathPerm( rsComm_t *rsComm, fileOpenInp_t *chkNVPathPermInp ) {
struct stat myFileStat;
int sysUid;
char tmpPath[MAX_NAME_LEN];
int len;
char *tmpPtr;
if ( chkNVPathPermInp->objPath[0] == '\0' ) {
std::stringstream msg;
msg << __FUNCTION__;
msg << " - Empty logical path.";
irods::log( LOG_ERROR, msg.str() );
return -1;
}
/* Need to match path's owner uid with sysUid */
sysUid = rsComm->clientUser.sysUid;
if ( sysUid < 0 ) {
/* have tried before */
return SYS_NO_PATH_PERMISSION;
}
else if ( sysUid == 0 ) {
sysUid = rsComm->clientUser.sysUid =
getUnixUid( rsComm->clientUser.userName );
if ( sysUid < 0 ) {
rsComm->clientUser.sysUid = sysUid;
return SYS_NO_PATH_PERMISSION;
}
}
rstrcpy( tmpPath, chkNVPathPermInp->fileName, MAX_NAME_LEN );
len = strlen( tmpPath );
irods::error stat_err;
while ( 1 ) {
irods::file_object_ptr file_obj(
new irods::file_object(
rsComm,
chkNVPathPermInp->objPath,
tmpPath,
chkNVPathPermInp->resc_hier_,
0, 0, 0 ) );
stat_err = fileStat( rsComm, file_obj, &myFileStat );
if ( stat_err.code() >= 0 ) {
break;
}
else if ( errno == EEXIST || getErrno( stat_err.code() ) == EEXIST ) {
/* go back */
tmpPtr = tmpPath + len;
while ( len > 0 ) {
len --;
if ( *tmpPtr == '/' ) {
*tmpPtr = '\0';
break;
}
tmpPtr--;
}
if ( len > 0 ) {
/* give it more tries */
continue;
}
else {
break;
}
}
else {
break;
}
}
if ( stat_err.code() < 0 ) {
return SYS_NO_PATH_PERMISSION;
}
if ( sysUid != ( int ) myFileStat.st_uid &&
( myFileStat.st_mode & S_IWOTH ) == 0 ) {
return SYS_NO_PATH_PERMISSION;
}
else {
return 0;
}
}
int fileChksum(
rsComm_t* rsComm,
char* objPath,
char* fileName,
char* rescHier,
char* orig_chksum,
char* chksumStr ) {
// =-=-=-=-=-=-=-
// capture server hashing settings
std::string svr_hash_scheme;
irods::server_properties& props = irods::server_properties::getInstance();
irods::error ret = props.get_property< std::string >(
DEFAULT_HASH_SCHEME_KW,
svr_hash_scheme );
std::string hash_scheme( irods::MD5_NAME );
if ( ret.ok() ) {
hash_scheme = svr_hash_scheme;
}
// make sure the read parameter is lowercased
std::transform(
hash_scheme.begin(),
hash_scheme.end(),
hash_scheme.begin(),
::tolower);
std::string svr_hash_policy;
ret = props.get_property< std::string >(
MATCH_HASH_POLICY_KW,
svr_hash_policy );
std::string hash_policy;
if ( ret.ok() ) {
hash_policy = svr_hash_policy;
}
// =-=-=-=-=-=-=-
// extract scheme from checksum string
std::string chkstr_scheme;
if ( orig_chksum ) {
ret = irods::get_hash_scheme_from_checksum(
orig_chksum,
chkstr_scheme );
if ( !ret.ok() ) {
//irods::log( PASS( ret ) );
}
}
// =-=-=-=-=-=-=-
// check the hash scheme against the policy
// if necessary
std::string final_scheme( hash_scheme );
if ( !chkstr_scheme.empty() ) {
if ( !hash_policy.empty() ) {
if ( irods::STRICT_HASH_POLICY == hash_policy ) {
if ( hash_scheme != chkstr_scheme ) {
return USER_HASH_TYPE_MISMATCH;
}
}
}
final_scheme = chkstr_scheme;
}
rodsLog(
LOG_DEBUG,
"fileChksum :: final_scheme [%s] chkstr_scheme [%s] svr_hash_policy [%s] hash_policy [%s]",
final_scheme.c_str(),
chkstr_scheme.c_str(),
svr_hash_policy.c_str(),
hash_policy.c_str() );
// =-=-=-=-=-=-=-
// call resource plugin to open file
irods::file_object_ptr file_obj(
new irods::file_object(
rsComm,
objPath,
fileName,
rescHier,
-1, 0, O_RDONLY ) ); // FIXME :: hack until this is better abstracted - JMC
ret = fileOpen( rsComm, file_obj );
if ( !ret.ok() ) {
int status = UNIX_FILE_OPEN_ERR - errno;
if ( ret.code() != DIRECT_ARCHIVE_ACCESS ) {
std::stringstream msg;
msg << "fileOpen failed for [";
msg << fileName;
msg << "]";
irods::log( PASSMSG( msg.str(), ret ) );
}
else {
status = ret.code();
}
return status;
}
// =-=-=-=-=-=-=-
// create a hasher object and init given a scheme
// if it is unsupported then default to md5
irods::Hasher hasher;
ret = irods::getHasher( final_scheme, hasher );
if ( !ret.ok() ) {
irods::log( PASS( ret ) );
irods::getHasher( irods::MD5_NAME, hasher );
}
// =-=-=-=-=-=-=-
// do an inital read of the file
char buffer[SVR_MD5_BUF_SZ];
irods::error read_err = fileRead(
rsComm,
file_obj,
buffer,
SVR_MD5_BUF_SZ );
int bytes_read = read_err.code();
// =-=-=-=-=-=-=-
// loop and update while there are still bytes to be read
#ifdef MD5_DEBUG
rodsLong_t total_bytes_read = 0; /* XXXX debug */
#endif
while ( read_err.ok() && bytes_read > 0 ) {
// =-=-=-=-=-=-=-
// debug statistics
#ifdef MD5_DEBUG
total_bytes_read += bytes_read;
#endif
// =-=-=-=-=-=-=-
// update hasher
hasher.update( std::string( buffer, bytes_read ) );
// =-=-=-=-=-=-=-
// read some more
read_err = fileRead( rsComm, file_obj, buffer, SVR_MD5_BUF_SZ );
if ( read_err.ok() ) {
bytes_read = read_err.code();
}
else {
std::stringstream msg;
msg << __FUNCTION__;
msg << " - Failed to read buffer from file: \"";
msg << fileName;
msg << "\"";
irods::error result = PASSMSG( msg.str(), read_err );
irods::log( result );
return result.code();
}
} // while
// =-=-=-=-=-=-=-
// close out the file
ret = fileClose( rsComm, file_obj );
if ( !ret.ok() ) {
irods::error err = PASSMSG( "error on close", ret );
irods::log( err );
}
// =-=-=-=-=-=-=-
// extract the digest from the hasher object
// and copy to outgoing string
std::string digest;
hasher.digest( digest );
strncpy( chksumStr, digest.c_str(), NAME_LEN );
// =-=-=-=-=-=-=-
// debug messaging
#ifdef MD5_DEBUG
rodsLog( LOG_NOTICE,
"fileChksum: chksum = %s, total_bytes_read = %lld",
chksumStr, total_bytes_read );
#endif
return 0;
}
/// =-=-=-=-=-=-=-
/// @brief function to query resource for chosen server to which to redirect
/// for a given operation
error resolve_resource_hierarchy(
const std::string& _oper,
rsComm_t* _comm,
dataObjInp_t* _data_obj_inp,
std::string& _out_hier ) {
// =-=-=-=-=-=-=-
// validate incoming parameters
if ( !_comm ) {
return ERROR(
SYS_INVALID_INPUT_PARAM,
"null comm pointer" );
}
else if ( !_data_obj_inp ) {
return ERROR(
SYS_INVALID_INPUT_PARAM,
"null data obj inp pointer" );
}
// =-=-=-=-=-=-=-
// cache the operation, as we may need to modify it
std::string oper = _oper;
// =-=-=-=-=-=-=-
// if this is a put operation then we do not have a first class object
resource_ptr resc;
file_object_ptr file_obj(
new file_object( ) );
// =-=-=-=-=-=-=-
// if this is a special collection then we need to get the hier
// pass that along and bail as it is not a data object, or if
// it is just a not-so-special collection then we continue with
// processing the operation, as this may be a create op
rodsObjStat_t *rodsObjStatOut = NULL;
int spec_stat = collStat( _comm, _data_obj_inp, &rodsObjStatOut );
file_obj->logical_path( _data_obj_inp->objPath );
if ( spec_stat >= 0 ) {
if ( rodsObjStatOut->specColl != NULL ) {
_out_hier = rodsObjStatOut->specColl->rescHier;
freeRodsObjStat( rodsObjStatOut );
return SUCCESS();
}
}
freeRodsObjStat( rodsObjStatOut );
// =-=-=-=-=-=-=-
// extract the resc name keyword from the conditional input
char* back_up_resc_name = getValByKey( &_data_obj_inp->condInput, BACKUP_RESC_NAME_KW );
char* dest_resc_name = getValByKey( &_data_obj_inp->condInput, DEST_RESC_NAME_KW );
char* default_resc_name = getValByKey( &_data_obj_inp->condInput, DEF_RESC_NAME_KW );
char* resc_name = getValByKey( &_data_obj_inp->condInput, RESC_NAME_KW );
// =-=-=-=-=-=-=-
// assign the keyword in an order, if it applies
char* key_word = 0;
if ( resc_name ) {
key_word = resc_name;
}
else if ( dest_resc_name ) {
key_word = dest_resc_name;
}
else if ( back_up_resc_name ) {
key_word = back_up_resc_name;
}
// =-=-=-=-=-=-=-
// call factory for given obj inp, get a file_object
error fac_err = file_object_factory( _comm, _data_obj_inp, file_obj );
// =-=-=-=-=-=-=-
// perform an open operation if create is not specificied ( thats all we have for now )
if ( OPEN_OPERATION == oper ||
WRITE_OPERATION == oper ) {
// =-=-=-=-=-=-=-
// factory has already been called, test for
// success before proceeding
if ( !fac_err.ok() ) {
std::stringstream msg;
msg << "resolve_resource_hierarchy :: failed in file_object_factory";
return PASSMSG( msg.str(), fac_err );
}
// =-=-=-=-=-=-=-
// consider force flag - we need to consider the default
// resc if -f is specified
char* force_flag = getValByKey( &_data_obj_inp->condInput, FORCE_FLAG_KW );
if ( force_flag &&
!key_word ) {
key_word = default_resc_name;
}
// =-=-=-=-=-=-=-
// attempt to resolve for an open
_out_hier = "";
error ret = resolve_hier_for_open_or_write(
_comm,
file_obj,
key_word,
oper,
_out_hier );
return ret;
}
else if ( CREATE_OPERATION == oper ) {
// =-=-=-=-=-=-=-
// include the default resc name if it applies
if ( !key_word && default_resc_name ) {
key_word = default_resc_name;
}
// =-=-=-=-=-=-=-
// if we have valid data objects then this could
// be actually an open rather than a pure create
error ret = SUCCESS();
if ( fac_err.ok() ) {
ret = resolve_hier_for_create_or_open(
_comm,
file_obj,
key_word,
_data_obj_inp,
_out_hier );
}
else {
// =-=-=-=-=-=-=-
// attempt to resolve for a create
ret = resolve_hier_for_create(
_comm,
file_obj,
key_word,
_data_obj_inp,
_out_hier );
}
return ret;
} // else
// =-=-=-=-=-=-=-
// should not get here
std::stringstream msg;
msg << "operation not supported ["
<< oper
<< "]";
return ERROR( -1, msg.str() );
} // resolve_resource_hierarchy
int _call_file_modified_for_modification(
rsComm_t* rsComm,
modDataObjMeta_t* modDataObjMetaInp ) {
int status = 0;
dataObjInfo_t *dataObjInfo;
keyValPair_t *regParam;
ruleExecInfo_t rei2;
memset( ( char* )&rei2, 0, sizeof( ruleExecInfo_t ) );
rei2.rsComm = rsComm;
if ( rsComm != NULL ) {
rei2.uoic = &rsComm->clientUser;
rei2.uoip = &rsComm->proxyUser;
}
rei2.doi = modDataObjMetaInp->dataObjInfo;
rei2.condInputData = modDataObjMetaInp->regParam;
regParam = modDataObjMetaInp->regParam;
dataObjInfo = modDataObjMetaInp->dataObjInfo;
if ( regParam->len == 0 ) {
return ( 0 );
}
if ( getValByKey( regParam, ALL_KW ) != NULL ) {
/* all copies */
dataObjInfo_t *dataObjInfoHead = NULL;
dataObjInfo_t *tmpDataObjInfo;
dataObjInp_t dataObjInp;
bzero( &dataObjInp, sizeof( dataObjInp ) );
rstrcpy( dataObjInp.objPath, dataObjInfo->objPath, MAX_NAME_LEN );
status = getDataObjInfoIncSpecColl( rsComm, &dataObjInp, &dataObjInfoHead );
if ( status < 0 ) {
rodsLog( LOG_NOTICE, "%s - Failed to get data objects.", __FUNCTION__ );
return status;
}
tmpDataObjInfo = dataObjInfoHead;
while ( tmpDataObjInfo != NULL ) {
if ( tmpDataObjInfo->specColl != NULL ) {
break;
}
irods::file_object_ptr file_obj(
new irods::file_object(
rsComm,
tmpDataObjInfo ) );
char* pdmo_kw = getValByKey( regParam, IN_PDMO_KW );
if ( pdmo_kw != NULL ) {
file_obj->in_pdmo( pdmo_kw );
}
irods::error ret = fileModified( rsComm, file_obj );
if ( !ret.ok() ) {
std::stringstream msg;
msg << __FUNCTION__;
msg << " - Failed to signal resource that the data object \"";
msg << tmpDataObjInfo->objPath;
msg << " was modified.";
ret = PASSMSG( msg.str(), ret );
irods::log( ret );
status = ret.code();
}
tmpDataObjInfo = tmpDataObjInfo->next;
}
freeAllDataObjInfo( dataObjInfoHead );
}
else {
irods::file_object_ptr file_obj(
new irods::file_object(
rsComm,
dataObjInfo ) );
char* pdmo_kw = getValByKey( regParam, IN_PDMO_KW );
if ( pdmo_kw != NULL ) {
file_obj->in_pdmo( pdmo_kw );
}
irods::error ret = fileModified( rsComm, file_obj );
if ( !ret.ok() ) {
std::stringstream msg;
msg << __FUNCTION__;
msg << " - Failed to signal the resource that the data object \"";
msg << dataObjInfo->objPath;
msg << "\" was modified.";
ret = PASSMSG( msg.str(), ret );
irods::log( ret );
status = ret.code();
}
}
return status;
}