// =-=-=-=-=-=-=- // local function which makes the call to chmod via the resource plugin int _rsFileChmod( rsComm_t* _comm, fileChmodInp_t* _chmod_inp ) { // =-=-=-=-=-=-=- // make the call to chmod via the resource plugin irods::file_object_ptr file_obj( new irods::file_object( _comm, _chmod_inp->objPath, _chmod_inp->fileName, _chmod_inp->rescHier, 0, 0, 0 ) ); irods::error chmod_err = fileChmod( _comm, file_obj, _chmod_inp->mode ); // =-=-=-=-=-=-=- // report errors if any if ( !chmod_err.ok() ) { std::stringstream msg; msg << "fileChmod failed for ["; msg << _chmod_inp->fileName; msg << "] to mode ["; msg << _chmod_inp->mode; msg << ""; irods::error err = PASSMSG( msg.str(), chmod_err ); irods::log( err ); } return chmod_err.code(); } //_rsChmod
int _call_file_modified_for_replica( rsComm_t* rsComm, regReplica_t* regReplicaInp ) { int status = 0; dataObjInfo_t* destDataObjInfo = regReplicaInp->destDataObjInfo; irods::file_object_ptr file_obj( new irods::file_object( rsComm, destDataObjInfo ) ); char* pdmo_kw = getValByKey( ®ReplicaInp->condInput, IN_PDMO_KW ); if ( pdmo_kw != NULL ) { file_obj->in_pdmo( pdmo_kw ); } irods::error ret = fileModified( rsComm, file_obj ); if ( !ret.ok() ) { std::stringstream msg; msg << __FUNCTION__; msg << " - Failed to signal resource that the data object \""; msg << destDataObjInfo->objPath; msg << "\" was registered"; ret = PASSMSG( msg.str(), ret ); irods::log( ret ); status = ret.code(); } return status; } // _call_file_modified_for_replica
// =-=-=-=-=-=-=- // local function which makes the call to truncate via the resource plugin int _rsFileTruncate( rsComm_t* _comm, fileOpenInp_t* _trunc_inp ) { // =-=-=-=-=-=-=- // make the call to rename via the resource plugin irods::file_object_ptr file_obj( new irods::file_object( _comm, _trunc_inp->objPath, _trunc_inp->fileName, _trunc_inp->resc_hier_, 0, 0, 0 ) ); file_obj->size( _trunc_inp->dataSize ); // =-=-=-=-=-=-=- // pass condInput file_obj->cond_input( _trunc_inp->condInput ); irods::error trunc_err = fileTruncate( _comm, file_obj ); // =-=-=-=-=-=-=- // report errors if any if ( !trunc_err.ok() ) { std::stringstream msg; msg << "fileTruncate for ["; msg << _trunc_inp->fileName; msg << "]"; msg << trunc_err.code(); irods::error err = PASSMSG( msg.str(), trunc_err ); irods::log( err ); } return trunc_err.code(); } // _rsFileTruncate
// =-=-=-=-=-=-=- // local function which makes the call to rename via the resource plugin int _rsFileRename( rsComm_t* _comm, fileRenameInp_t* _rename_inp, fileRenameOut_t** _rename_out, rodsServerHost_t* _server_host ) { // =-=-=-=-=-=-=- // FIXME: need to check resource permission and vault permission // when RCAT is available // mkDirForFilePath( _comm, "/", _rename_inp->newFileName, getDefDirMode () ); - The actual file path depends on the resource // =-=-=-=-=-=-=- // make the call to rename via the resource plugin irods::file_object_ptr file_obj( new irods::file_object( _comm, _rename_inp->objPath, _rename_inp->oldFileName, _rename_inp->rescHier, 0, 0, 0 ) ); irods::error rename_err = fileRename( _comm, file_obj, _rename_inp->newFileName ); // =-=-=-=-=-=-=- // report errors if any if ( !rename_err.ok() ) { std::stringstream msg; msg << "fileRename failed for ["; msg << _rename_inp->oldFileName; msg << "] to ["; msg << _rename_inp->newFileName; msg << "]"; irods::error err = PASSMSG( msg.str(), rename_err ); irods::log( err ); } // =-=-=-=-=-=-=- // compare fco phy path and old file name // if they differ then repave for next call if ( file_obj->physical_path() == _rename_inp->oldFileName ) { file_obj->physical_path( _rename_inp->newFileName ); } // =-=-=-=-=-=-=- // percolate possible change in phy path up ( *_rename_out ) = ( fileRenameOut_t* ) malloc( sizeof( fileRenameOut_t ) ); strncpy( ( *_rename_out )->file_name, file_obj->physical_path().c_str(), MAX_NAME_LEN ); return rename_err.code(); } // _rsFileRename
// =-=-=-=-=-=-=- // local function to handle call to stat via resource plugin int _rsFileLseek( rsComm_t* _comm, fileLseekInp_t* _lseek_inp, fileLseekOut_t** _lseek_out ) { // =-=-=-=-=-=-=- // make call to lseek via resource plugin irods::file_object_ptr file_obj( new irods::file_object( _comm, FileDesc[_lseek_inp->fileInx].objPath, FileDesc[_lseek_inp->fileInx].fileName, FileDesc[_lseek_inp->fileInx].rescHier, FileDesc[_lseek_inp->fileInx].fd, 0, 0 ) ); irods::error lseek_err = fileLseek( _comm, file_obj, _lseek_inp->offset, _lseek_inp->whence ); // =-=-=-=-=-=-=- // handle error conditions and log if ( !lseek_err.ok() ) { std::stringstream msg; msg << "lseek failed for ["; msg << FileDesc[_lseek_inp->fileInx].fileName; msg << "]"; irods::error ret_err = PASSMSG( msg.str(), lseek_err ); irods::log( ret_err ); return lseek_err.code(); } else { ( *_lseek_out ) = ( fileLseekOut_t* )malloc( sizeof( fileLseekOut_t ) ); memset( ( *_lseek_out ), 0, sizeof( fileLseekOut_t ) ); ( *_lseek_out )->offset = lseek_err.code(); return 0; } } // _rsFileLseek
int _rsFileRead( rsComm_t* _comm, fileReadInp_t* _read_inp, bytesBuf_t* _read_bbuf ) { // =-=-=-=-=-=-=- // XXXX need to check resource permission and vault permission // when RCAT is available // =-=-=-=-=-=-=- // call resource plugin for POSIX read irods::file_object_ptr file_obj( new irods::file_object( _comm, FileDesc[_read_inp->fileInx].objPath, FileDesc[_read_inp->fileInx].fileName, FileDesc[_read_inp->fileInx].rescHier, FileDesc[_read_inp->fileInx].fd, 0, 0 ) ); irods::error ret = fileRead( _comm, file_obj, _read_bbuf->buf, _read_inp->len ); // =-=-=-=-=-=-= // log an error if the read failed, // pass long read error if ( !ret.ok() ) { std::stringstream msg; msg << "fileRead failed for "; msg << file_obj->physical_path(); msg << "]"; irods::error err = PASSMSG( msg.str(), ret ); irods::log( err ); } else { _read_bbuf->len = ret.code(); } return ret.code(); } // _rsFileRead
// =-=-=-=-=-=-=- // local function to handle call to stat via resource plugin int _rsFileStat( rsComm_t* _comm, fileStatInp_t* _stat_inp, rodsStat_t** _stat_out ) { struct stat myFileStat; memset( &myFileStat, 0, sizeof( myFileStat ) ); // =-=-=-=-=-=-=- // make call to stat via resource plugin irods::file_object_ptr file_obj( new irods::file_object( _comm, _stat_inp->objPath, _stat_inp->fileName, _stat_inp->rescId, 0, 0, 0 ) ); irods::error stat_err = fileStat( _comm, file_obj, &myFileStat ); // =-=-=-=-=-=-=- // log error if necessary if ( !stat_err.ok() ) { // irods::log(LOG_ERROR, stat_err.result()); return stat_err.code(); } // =-=-=-=-=-=-=- // convert unix stat struct to an irods stat struct *_stat_out = ( rodsStat_t* )malloc( sizeof( rodsStat_t ) ); int status = statToRodsStat( *_stat_out, &myFileStat ); // =-=-=-=-=-=-=- // manage error if necessary if ( status < 0 ) { free( *_stat_out ); *_stat_out = NULL; } return status; } // _rsFileStat
int _rsRegDataObj( rsComm_t *rsComm, dataObjInfo_t *dataObjInfo ) { #ifdef RODS_CAT int status; irods::error ret; status = chlRegDataObj( rsComm, dataObjInfo ); if ( status < 0 ) { char* sys_error; char* rods_error = rodsErrorName( status, &sys_error ); std::stringstream msg; msg << __FUNCTION__; msg << " - Failed to register data object \"" << dataObjInfo->objPath << "\""; msg << " - " << rods_error << " " << sys_error; ret = ERROR( status, msg.str() ); irods::log( ret ); } else { irods::file_object_ptr file_obj( new irods::file_object( rsComm, dataObjInfo ) ); ret = fileRegistered( rsComm, file_obj ); if ( !ret.ok() ) { std::stringstream msg; msg << __FUNCTION__; msg << " - Failed to signal resource that the data object \""; msg << dataObjInfo->objPath; msg << "\" was registered"; ret = PASSMSG( msg.str(), ret ); irods::log( ret ); status = ret.code(); } } return status; #else return SYS_NO_RCAT_SERVER_ERR; #endif }
int _l3DataPutSingleBuf( rsComm_t *rsComm, int l1descInx, dataObjInp_t *dataObjInp, bytesBuf_t *dataObjInpBBuf ) { int status = 0; dataObjInfo_t *myDataObjInfo = L1desc[l1descInx].dataObjInfo; int bytesWritten = l3FilePutSingleBuf( rsComm, l1descInx, dataObjInpBBuf ); if ( bytesWritten >= 0 ) { if ( L1desc[l1descInx].replStatus == NEWLY_CREATED_COPY && myDataObjInfo->specColl == NULL && L1desc[l1descInx].remoteZoneHost == NULL ) { /* the check for remoteZoneHost host is not needed because * the put would have done in the remote zone. But it make * the code easier to read (similar ro copy). */ status = svrRegDataObj( rsComm, myDataObjInfo ); if ( status < 0 ) { rodsLog( LOG_NOTICE, "l3DataPutSingleBuf: rsRegDataObj for %s failed, status = %d", myDataObjInfo->objPath, status ); if ( status != CATALOG_ALREADY_HAS_ITEM_BY_THAT_NAME ) { l3Unlink( rsComm, myDataObjInfo ); } return status; } else { myDataObjInfo->replNum = status; } } /* myDataObjInfo->dataSize = bytesWritten; update size problem */ if ( bytesWritten == 0 && myDataObjInfo->dataSize > 0 ) { /* overwrite with 0 len file */ L1desc[l1descInx].bytesWritten = 1; } else { L1desc[l1descInx].bytesWritten = bytesWritten; } // special case of zero length files, trigger the fileModified // operation for execution of coordinating resource logic if ( 0 == dataObjInp->dataSize ) { irods::file_object_ptr file_obj( new irods::file_object( rsComm, myDataObjInfo ) ); char* pdmo_kw = getValByKey( &myDataObjInfo->condInput, IN_PDMO_KW ); if ( pdmo_kw != NULL ) { file_obj->in_pdmo( pdmo_kw ); } irods::error ret = fileModified( rsComm, file_obj ); if ( !ret.ok() ) { std::stringstream msg; msg << "fileModified failed for ["; msg << myDataObjInfo->objPath; msg << "]"; ret = PASSMSG( msg.str(), ret ); irods::log( ret ); status = ret.code(); } } } L1desc[l1descInx].dataSize = dataObjInp->dataSize; return bytesWritten; }
int rsDataObjOpen( rsComm_t *rsComm, dataObjInp_t *dataObjInp ) { int status, l1descInx; int remoteFlag; rodsServerHost_t *rodsServerHost; remoteFlag = getAndConnRemoteZone( rsComm, dataObjInp, &rodsServerHost, REMOTE_OPEN ); if ( remoteFlag < 0 ) { return remoteFlag; } else if ( remoteFlag == REMOTE_HOST ) { openStat_t *openStat = NULL; status = rcDataObjOpenAndStat( rodsServerHost->conn, dataObjInp, &openStat ); if ( status < 0 ) { return status; } l1descInx = allocAndSetL1descForZoneOpr( status, dataObjInp, rodsServerHost, openStat ); if ( openStat != NULL ) { free( openStat ); } return l1descInx; } else { // dataObjInfo_t linked list dataObjInfo_t *dataObjInfoHead = NULL; // resource hierarchy std::string hier; // =-=-=-=-=-=-=- // determine the resource hierarchy if one is not provided if ( getValByKey( &dataObjInp->condInput, RESC_HIER_STR_KW ) == NULL ) { irods::error ret = irods::resolve_resource_hierarchy( irods::OPEN_OPERATION, rsComm, dataObjInp, hier, &dataObjInfoHead ); if (ret.ok()) { // =-=-=-=-=-=-=- // we resolved the redirect and have a host, set the hier str for subsequent // api calls, etc. addKeyVal( &dataObjInp->condInput, RESC_HIER_STR_KW, hier.c_str() ); } } else { // file object for file_object_factory irods::file_object_ptr file_obj( new irods::file_object() ); // get resource hierarchy from condInput hier = getValByKey( &dataObjInp->condInput, RESC_HIER_STR_KW ); // get replicas vector populated irods::error fac_err = irods::file_object_factory(rsComm, dataObjInp, file_obj, &dataObjInfoHead); }// if ( getValByKey( &dataObjInp->condInput, RESC_HIER_STR_KW ) == NULL ) l1descInx = _rsDataObjOpen( rsComm, dataObjInp, dataObjInfoHead ); } return l1descInx; }
int main(int argc, char *argv[]) { QCoreApplication a(argc, argv); QString file_path = "/home/eveil/src/dev/qt-json/Load-JSON-deep/test.json"; QFile file_obj(file_path); if(!file_obj.open(QIODevice::ReadOnly)){ qDebug()<< "Failed to open file: " << file_path; exit(1); } QTextStream file_text(&file_obj); QString json_string; json_string = file_text.readAll(); file_obj.close(); QByteArray json_bytes = json_string.toLocal8Bit(); auto json_doc = QJsonDocument::fromJson(json_bytes); if(json_doc.isNull()){ qDebug()<< "Failed to creat json doc from: " <<endl<<json_bytes; exit(2); } qDebug() << json_doc; if(!json_doc.isObject()){ qDebug() << "json doc is not object."; exit(3); } QJsonObject root_obj=json_doc.object(); QVariantMap root_map=root_obj.toVariantMap(); QVariantMap stats=root_map["stats"].toMap(); QVariantList inv=root_map["inventory"].toList(); if(stats.isEmpty()){ qDebug() << "The stats object is empty."; exit(4); } if(inv.isEmpty()){ qDebug() << "The inventory array is empty"; } // iterate a json object's members: QStringList key_list = stats.keys(); for(int i=0; i < key_list.count(); ++i){ QString key=key_list.at(i); int stat_val = stats[key.toLocal8Bit()].toInt(); qDebug() << key << ": " << stat_val; } qDebug() << "Inventory:"; for(int i=0; i < inv.count(); ++i){ qDebug() << inv.at(i).toString(); } return a.exec(); }
// =-=-=-=-=-=-=-= // _rsFileSyncToArch - this the local version of rsFileSyncToArch. int _rsFileSyncToArch( rsComm_t* _comm, fileStageSyncInp_t* _sync_inp, fileSyncOut_t** _sync_out ) { // =-=-=-=-=-=-=- // XXXX need to check resource permission and vault permission // when RCAT is available // =-=-=-=-=-=-=- // prep if ( _sync_inp->objPath[0] == '\0' ) { std::stringstream msg; msg << __FUNCTION__; msg << " - Empty logical path."; irods::log( LOG_ERROR, msg.str() ); return SYS_INVALID_INPUT_PARAM; } // =-=-=-=-=-=-=- // make call to synctoarch via resource plugin irods::file_object_ptr file_obj( new irods::file_object( _comm, _sync_inp->objPath, _sync_inp->filename, "", 0, _sync_inp->mode, _sync_inp->flags ) ); file_obj->resc_hier( _sync_inp->rescHier ); // =-=-=-=-=-=-=- // pass condInput file_obj->cond_input( _sync_inp->condInput ); irods::error sync_err = fileSyncToArch( _comm, file_obj, _sync_inp->cacheFilename ); if ( !sync_err.ok() ) { if ( getErrno( sync_err.code() ) == ENOENT ) { // =-=-=-=-=-=-=- // the directory does not exist, lets make one int status = mkDirForFilePath( _comm, 0, _sync_inp->filename, _sync_inp->rescHier, getDefDirMode() ); if ( status < 0 ) { rodsLog( LOG_ERROR, "mkDirForFilePath failed in _rsFileSyncToArch with status %d", status ); return status; } } else if ( getErrno( sync_err.code() ) == EEXIST ) { // =-=-=-=-=-=-=- // an empty dir may be there, make the call to rmdir via the resource plugin irods::collection_object_ptr coll_obj( new irods::collection_object( _sync_inp->filename, _sync_inp->rescHier, 0, 0 ) ); coll_obj->cond_input( _sync_inp->condInput ); irods::error rmdir_err = fileRmdir( _comm, coll_obj ); if ( !rmdir_err.ok() ) { std::stringstream msg; msg << "fileRmdir failed for ["; msg << _sync_inp->filename; msg << "]"; irods::error err = PASSMSG( msg.str(), sync_err ); irods::log( err ); } } else { std::stringstream msg; msg << "fileSyncToArch failed for ["; msg << _sync_inp->filename; msg << "]"; irods::error err = PASSMSG( msg.str(), sync_err ); irods::log( err ); return sync_err.code(); } // =-=-=-=-=-=-=- // make call to synctoarch via resource plugin sync_err = fileSyncToArch( _comm, file_obj, _sync_inp->cacheFilename ); if ( !sync_err.ok() ) { std::stringstream msg; msg << "fileSyncToArch failed for ["; msg << _sync_inp->filename; msg << "]"; msg << sync_err.code(); irods::error err = PASSMSG( msg.str(), sync_err ); irods::log( err ); } } // if !sync_err.ok() // =-=-=-=-=-=-=- // has the file name has changed? if ( *_sync_out ) { rstrcpy( ( *_sync_out )->file_name, file_obj->physical_path().c_str(), MAX_NAME_LEN ); } return sync_err.code(); } // _rsFileSyncToArch
int _rsFileGet( rsComm_t* _comm, fileOpenInp_t* _get_inp, bytesBuf_t* _get_buf ) { int fd; int len; len = _get_inp->dataSize; fd = _rsFileOpen( _comm, _get_inp ); if ( fd < 0 ) { rodsLog( LOG_NOTICE, "_rsFileGet: fileGet for %s, status = %d", _get_inp->fileName, fd ); return fd; } if ( _get_buf->buf == NULL ) { _get_buf->buf = malloc( len ); } irods::file_object_ptr file_obj( new irods::file_object( _comm, _get_inp->objPath, _get_inp->fileName, _get_inp->resc_hier_, fd, _get_inp->mode, _get_inp->flags ) ); // =-=-=-=-=-=-=- // pass condInput file_obj->cond_input( _get_inp->condInput ); irods::error read_err = fileRead( _comm, file_obj, _get_buf->buf, len ); int bytes_read = read_err.code(); if ( bytes_read != len ) { if ( bytes_read >= 0 ) { _get_buf->len = bytes_read; } else { std::stringstream msg; msg << "fileRead failed for ["; msg << _get_inp->fileName; msg << "]"; irods::error ret_err = PASSMSG( msg.str(), read_err ); irods::log( ret_err ); } } else { _get_buf->len = bytes_read; } // =-=-=-=-=-=-=- // call resource plugin close irods::error close_err = fileClose( _comm, file_obj ); if ( !close_err.ok() ) { irods::error err = PASSMSG( "error on close", close_err ); irods::log( err ); } return bytes_read; } // _rsFileGet
std::string Java_Compiler_Dump_Algorithm::process ( boost::uint32_t compiler_id, std::string filename ) { #ifdef LIBREVERSE_DEBUG Trace::write_Trace ( TraceArea::GRNN_OPTIMIZER, TraceLevel::DETAIL, "Entering Java_Compiler_Dump_Algorithm::process" ); #endif /* LIBREVERSE_DEBUG */ std::stringstream output; try { reverse::io_types::File_ID::ptr_t file_obj ( new reverse::io::File_ID ( filename ) ); reverse::java_module::Reader r_obj ( file_obj ); r_obj.read_Class_Header(); reverse::data_types::memory_map::ptr_t mem_ptr = r_obj.get_memory_map(); output << boost::format(" <%1%>") % classifier::Java_Input_Tag_Names::TAG_FILE << std::endl; output << boost::format ( " <%1%>%2%</%3%>" ) % classifier::Java_Input_Tag_Names::TAG_TARGET_ID % compiler_id % classifier::Java_Input_Tag_Names::TAG_TARGET_ID << std::endl; output << boost::format ( " <%1%>%2%</%3%>" ) % classifier::Java_Input_Tag_Names::TAG_FILESIZE % ( mem_ptr->size() ) % classifier::Java_Input_Tag_Names::TAG_FILESIZE << std::endl; // Get class header reverse::java_types::Class_Header::ptr_t hdr_ptr = r_obj.get_Header(); // Grab data std::vector<float> stats(12,0); collect_Constant_Pool_Stats ( stats, hdr_ptr->get_Constant_Pool_Begin(), hdr_ptr->get_Constant_Pool_End() ); // Add entry to output file output << boost::format ( " <%1%>%2%.%3%</%4%>" ) % classifier::Java_Input_Tag_Names::TAG_VERSION % hdr_ptr->get_Major_Version() % hdr_ptr->get_Minor_Version() % classifier::Java_Input_Tag_Names::TAG_VERSION << std::endl; output << boost::format ( " <%1%>%2%</%3%>" ) % classifier::Java_Input_Tag_Names::TAG_THIS_INDEX % hdr_ptr->get_This_Class() % classifier::Java_Input_Tag_Names::TAG_THIS_INDEX << std::endl; output << boost::format ( " <%1%>%2%</%3%>" ) % classifier::Java_Input_Tag_Names::TAG_SUPER_INDEX % hdr_ptr->get_Super_Class() % classifier::Java_Input_Tag_Names::TAG_SUPER_INDEX << std::endl; output << boost::format ( " <%1%>%2%</%3%>" ) % classifier::Java_Input_Tag_Names::TAG_CONSTANT_POOL_COUNT % hdr_ptr->get_Constant_Pool_Count() % classifier::Java_Input_Tag_Names::TAG_CONSTANT_POOL_COUNT << std::endl; float constant_pool_total = hdr_ptr->get_Constant_Pool_Count(); print_Constant_Pool_Stats ( output, stats, constant_pool_total ); output << boost::format ( " <%1%>%2%</%3%>" ) % classifier::Java_Input_Tag_Names::TAG_FIELD_COUNT % hdr_ptr->get_Field_Count() % classifier::Java_Input_Tag_Names::TAG_FIELD_COUNT << std::endl; output << boost::format ( " <%1%>%2%</%3%>" ) % classifier::Java_Input_Tag_Names::TAG_METHOD_COUNT % hdr_ptr->get_Method_Count() % classifier::Java_Input_Tag_Names::TAG_METHOD_COUNT << std::endl; output << boost::format(" </%1%>" ) % classifier::Java_Input_Tag_Names::TAG_FILE << std::endl; } catch ( std::exception &e ) { #ifdef LIBREVERSE_DEBUG Trace::write_Trace ( TraceArea::GRNN_OPTIMIZER, TraceLevel::ERROR, boost::str ( boost::format("(ERROR) Cannot read %1%. Skipping the file. ") % filename ) ); Trace::write_Trace ( TraceArea::GRNN_OPTIMIZER, TraceLevel::ERROR, e.what() ); #endif /* LIBREVERSE_DEBUG */ } #ifdef LIBREVERSE_DEBUG Trace::write_Trace ( TraceArea::GRNN_OPTIMIZER, TraceLevel::DETAIL, "Exiting Java_Compiler_Dump_Algorithm::process" ); #endif /* LIBREVERSE_DEBUG */ return output.str(); }
int main() { time_t start_time = time(NULL); std::string data_path("D:/Zmisc/Github/NLP/Naive-Bayes-Document-Classifier-master/trial/reuters_modified/"); //change this path according to your local data path std::ifstream file_cate((data_path+std::string("cats.txt")).c_str()); //if file can't be opened, we exit if(!file_cate.good()) { std::cerr << "Can't open cats.txt to read.\n"; //exit(1); which is the same as return EXIT_FAILURE; } std::string line; std::string token; std::string word1; std::string word2; std::string file_name; std::string min_cat; std::string train("training"); std::string tes("test"); //using the tree map implementation with O(log n) time for insertion, search, deletion etc //because there are no hash table implementations in c++ stl std::map<std::string,std::string > test_map; //test file to category map std::map<std::string,std::string > train_map; //train file to category map std::map<std::string,std::string >::iterator fi_itr; std::map<std::string,std::string >::const_iterator cfi_itr; std::map<std::string,int > cate_num_docs_map; std::map<std::string,int > cate_num_docs_test_map; std::map<std::string,int >::iterator itr; std::map<std::string,int >::const_iterator citr; std::set<std::string> words_set; std::set<std::string>::iterator set_itr; std::set<std::string>::const_iterator cset_itr; std::map<std::string, std::map<std::string,unsigned int> > word_cates_num_map; std::map<std::string, std::map<std::string,unsigned int> >::iterator big_itr; std::map<std::string, std::map<std::string,unsigned int> >::const_iterator cbig_itr; std::map<std::string,unsigned int>::iterator small_itr; std::map<std::string,unsigned int>::const_iterator csmall_itr; //Create a map with a word as the key and a map as the value // in the inner map the category as key and number of documents in that category where it occurs as value std::map<std::string, std::vector<unsigned int> > cates_results; std::map<std::string, std::vector<unsigned int> >::iterator res_itr; std::map<std::string, std::vector<unsigned int> >::const_iterator cres_itr; unsigned int len_train; unsigned int counter; unsigned int error_counter; unsigned int len_test; int nct; double time_diff; double minimum_neg_log_prob; double neg_log_prob; double ratio; //Format of each line of categs file - filename cate1 cate 2(if any) cate3(if any) while(getline(file_cate,line)) { counter = 0; std::istringstream iss(line); while(iss >> token) { ++counter; if(counter==1) word1 = token; else if(counter==2) word2 = token; } if(counter>2) //we only consider files which are in 1 category continue; if(word1.compare(0,4,tes)==0) { file_name = word1.substr(5); test_map[file_name] = word2; ++cate_num_docs_test_map[word2]; } else if(word1.compare(0,8,train)==0) { file_name = word1.substr(9); train_map[file_name] = word2; ++cate_num_docs_map[word2]; } } file_cate.close(); //Removing categories with less than 21 files in either the training set or test set prune_rare_categories(cate_num_docs_map,cate_num_docs_test_map); for(cfi_itr = train_map.begin();cfi_itr!=train_map.end();++cfi_itr) { itr = cate_num_docs_map.find(cfi_itr->second); //File belongs to a low category if(itr==cate_num_docs_map.end()) continue; file_name = data_path+std::string("training/")+cfi_itr->first; std::ifstream file_obj(file_name.c_str()); //if file can't be opened, we exit if(!file_obj.good()) { std::cerr << "Can't open training file to read.\n"; //exit(1); which is the same as return EXIT_FAILURE; } words_set.clear(); while(file_obj >> token) words_set.insert(token); for(cset_itr = words_set.begin();cset_itr!=words_set.end();++cset_itr) { big_itr = word_cates_num_map.find(*cset_itr); ++word_cates_num_map[*cset_itr][cfi_itr->second]; } file_obj.close(); ++len_train; } time_diff = difftime(time(NULL),start_time); std::cout << "The Classifier is trained and it took "<< time_diff << " seconds.\n" ; start_time = time(NULL); //Initialize cates_results for(citr=cate_num_docs_map.begin();citr!=cate_num_docs_map.end();++citr) { cates_results[citr->first].push_back(0); cates_results[citr->first].push_back(0); cates_results[citr->first].push_back(0); cates_results[citr->first].push_back(0); } for(cfi_itr = test_map.begin();cfi_itr!=test_map.end();++cfi_itr) { itr = cate_num_docs_map.find(cfi_itr->second); //File belongs to a low category if(itr==cate_num_docs_map.end()) continue; file_name = data_path+std::string("test/")+cfi_itr->first; std::ifstream file_obj(file_name.c_str()); //if file can't be opened, we exit if(!file_obj.good()) { std::cerr << "Can't open test file to read.\n"; //exit(1); which is the same as return EXIT_FAILURE; } words_set.clear(); while(file_obj >> token) words_set.insert(token); minimum_neg_log_prob=1000000000; min_cat = ""; for(citr=cate_num_docs_map.begin();citr!=cate_num_docs_map.end();++citr) { neg_log_prob = -log(citr->second/(len_train*1.0)); for(cbig_itr = word_cates_num_map.begin(); cbig_itr!=word_cates_num_map.end();++cbig_itr) { set_itr = words_set.find(cbig_itr->first); nct = word_cates_num_map[cbig_itr->first][citr->first]; ratio = (nct+1)/(citr->second+2.0); if(set_itr!=words_set.end()) neg_log_prob-= log(ratio); else neg_log_prob-=log(1-ratio); } if(minimum_neg_log_prob>neg_log_prob) { min_cat = citr->first; minimum_neg_log_prob = neg_log_prob; } } file_obj.close(); ++len_test; if(min_cat!=cfi_itr->second) ++error_counter; for(cres_itr = cates_results.begin();cres_itr!=cates_results.end();++cres_itr) { word2 = cres_itr->first; if(word2==min_cat) { if(word2==cfi_itr->second) ++cates_results[word2][0]; else ++cates_results[word2][1]; } else { if(word2==cfi_itr->second) ++cates_results[word2][2]; else ++cates_results[word2][3]; } } } std::cout << "The fraction of errors is " << error_counter/(len_test*1.0) << "\n"; //Evaluation by finer measures f_measure(cates_results); time_diff = difftime(time(NULL),start_time); std::cout << "The Classifier has run and it took "<< time_diff << " seconds.\n" ; return 0; }
// =-=-=-=-=-=-=- // local implementation of put int _rsFilePut( rsComm_t* _comm, fileOpenInp_t* _put_inp, bytesBuf_t* _put_bbuf, rodsServerHost_t* _server_host ) { int fd = 0; // =-=-=-=-=-=-=- // NOTE:: this test does not seem to work for i86 solaris if ( ( _put_inp->otherFlags & FORCE_FLAG ) != 0 ) { // =-=-=-=-=-=-=- // create one if it does not exist */ _put_inp->flags |= O_CREAT; fd = _rsFileOpen( _comm, _put_inp ); } else { fd = _rsFileCreate( _comm, _put_inp, _server_host ); } // else // =-=-=-=-=-=-=- // log, error if any if ( fd < 0 ) { if ( getErrno( fd ) == EEXIST ) { rodsLog( LOG_DEBUG1, "_rsFilePut: filePut for %s, status = %d", _put_inp->fileName, fd ); } else if ( fd != DIRECT_ARCHIVE_ACCESS ) { rodsLog( LOG_DEBUG, "_rsFilePut: filePut for %s, status = %d", _put_inp->fileName, fd ); } return ( fd ); } // =-=-=-=-=-=-=- // call write for resource plugin irods::file_object_ptr file_obj( new irods::file_object( _comm, _put_inp->objPath, _put_inp->fileName, _put_inp->resc_hier_, fd, 0, 0 ) ); file_obj->in_pdmo( _put_inp->in_pdmo ); file_obj->cond_input( _put_inp->condInput ); irods::error write_err = fileWrite( _comm, file_obj, _put_bbuf->buf, _put_bbuf->len ); int write_code = write_err.code(); // =-=-=-=-=-=-=- // log errors, if any if ( write_code != _put_bbuf->len ) { if ( write_code >= 0 ) { std::stringstream msg; msg << "fileWrite failed for ["; msg << _put_inp->fileName; msg << "] towrite ["; msg << _put_bbuf->len; msg << "] written ["; msg << write_code << "]"; irods::error err = PASSMSG( msg.str(), write_err ); irods::log( err ); write_code = SYS_COPY_LEN_ERR; } else { std::stringstream msg; msg << "fileWrite failed for ["; msg << _put_inp->fileName; msg << "]"; irods::error err = PASSMSG( msg.str(), write_err ); irods::log( err ); } } // =-=-=-=-=-=-=- // close up after ourselves irods::error close_err = fileClose( _comm, file_obj ); if ( !close_err.ok() ) { irods::error err = PASSMSG( "error on close", close_err ); irods::log( err ); } // =-=-=-=-=-=-=- // return 'write_err code' as this includes this implementation // assumes we are returning the size of the file 'put' via fileWrite return write_code; } // _rsFilePut
int _rsChkNVPathPerm( rsComm_t *rsComm, fileOpenInp_t *chkNVPathPermInp ) { struct stat myFileStat; int sysUid; char tmpPath[MAX_NAME_LEN]; int len; char *tmpPtr; if ( chkNVPathPermInp->objPath[0] == '\0' ) { std::stringstream msg; msg << __FUNCTION__; msg << " - Empty logical path."; irods::log( LOG_ERROR, msg.str() ); return -1; } /* Need to match path's owner uid with sysUid */ sysUid = rsComm->clientUser.sysUid; if ( sysUid < 0 ) { /* have tried before */ return SYS_NO_PATH_PERMISSION; } else if ( sysUid == 0 ) { sysUid = rsComm->clientUser.sysUid = getUnixUid( rsComm->clientUser.userName ); if ( sysUid < 0 ) { rsComm->clientUser.sysUid = sysUid; return SYS_NO_PATH_PERMISSION; } } rstrcpy( tmpPath, chkNVPathPermInp->fileName, MAX_NAME_LEN ); len = strlen( tmpPath ); irods::error stat_err; while ( 1 ) { irods::file_object_ptr file_obj( new irods::file_object( rsComm, chkNVPathPermInp->objPath, tmpPath, chkNVPathPermInp->resc_hier_, 0, 0, 0 ) ); stat_err = fileStat( rsComm, file_obj, &myFileStat ); if ( stat_err.code() >= 0 ) { break; } else if ( errno == EEXIST || getErrno( stat_err.code() ) == EEXIST ) { /* go back */ tmpPtr = tmpPath + len; while ( len > 0 ) { len --; if ( *tmpPtr == '/' ) { *tmpPtr = '\0'; break; } tmpPtr--; } if ( len > 0 ) { /* give it more tries */ continue; } else { break; } } else { break; } } if ( stat_err.code() < 0 ) { return SYS_NO_PATH_PERMISSION; } if ( sysUid != ( int ) myFileStat.st_uid && ( myFileStat.st_mode & S_IWOTH ) == 0 ) { return SYS_NO_PATH_PERMISSION; } else { return 0; } }
int fileChksum( rsComm_t* rsComm, char* objPath, char* fileName, char* rescHier, char* orig_chksum, char* chksumStr ) { // =-=-=-=-=-=-=- // capture server hashing settings std::string svr_hash_scheme; irods::server_properties& props = irods::server_properties::getInstance(); irods::error ret = props.get_property< std::string >( DEFAULT_HASH_SCHEME_KW, svr_hash_scheme ); std::string hash_scheme( irods::MD5_NAME ); if ( ret.ok() ) { hash_scheme = svr_hash_scheme; } // make sure the read parameter is lowercased std::transform( hash_scheme.begin(), hash_scheme.end(), hash_scheme.begin(), ::tolower); std::string svr_hash_policy; ret = props.get_property< std::string >( MATCH_HASH_POLICY_KW, svr_hash_policy ); std::string hash_policy; if ( ret.ok() ) { hash_policy = svr_hash_policy; } // =-=-=-=-=-=-=- // extract scheme from checksum string std::string chkstr_scheme; if ( orig_chksum ) { ret = irods::get_hash_scheme_from_checksum( orig_chksum, chkstr_scheme ); if ( !ret.ok() ) { //irods::log( PASS( ret ) ); } } // =-=-=-=-=-=-=- // check the hash scheme against the policy // if necessary std::string final_scheme( hash_scheme ); if ( !chkstr_scheme.empty() ) { if ( !hash_policy.empty() ) { if ( irods::STRICT_HASH_POLICY == hash_policy ) { if ( hash_scheme != chkstr_scheme ) { return USER_HASH_TYPE_MISMATCH; } } } final_scheme = chkstr_scheme; } rodsLog( LOG_DEBUG, "fileChksum :: final_scheme [%s] chkstr_scheme [%s] svr_hash_policy [%s] hash_policy [%s]", final_scheme.c_str(), chkstr_scheme.c_str(), svr_hash_policy.c_str(), hash_policy.c_str() ); // =-=-=-=-=-=-=- // call resource plugin to open file irods::file_object_ptr file_obj( new irods::file_object( rsComm, objPath, fileName, rescHier, -1, 0, O_RDONLY ) ); // FIXME :: hack until this is better abstracted - JMC ret = fileOpen( rsComm, file_obj ); if ( !ret.ok() ) { int status = UNIX_FILE_OPEN_ERR - errno; if ( ret.code() != DIRECT_ARCHIVE_ACCESS ) { std::stringstream msg; msg << "fileOpen failed for ["; msg << fileName; msg << "]"; irods::log( PASSMSG( msg.str(), ret ) ); } else { status = ret.code(); } return status; } // =-=-=-=-=-=-=- // create a hasher object and init given a scheme // if it is unsupported then default to md5 irods::Hasher hasher; ret = irods::getHasher( final_scheme, hasher ); if ( !ret.ok() ) { irods::log( PASS( ret ) ); irods::getHasher( irods::MD5_NAME, hasher ); } // =-=-=-=-=-=-=- // do an inital read of the file char buffer[SVR_MD5_BUF_SZ]; irods::error read_err = fileRead( rsComm, file_obj, buffer, SVR_MD5_BUF_SZ ); int bytes_read = read_err.code(); // =-=-=-=-=-=-=- // loop and update while there are still bytes to be read #ifdef MD5_DEBUG rodsLong_t total_bytes_read = 0; /* XXXX debug */ #endif while ( read_err.ok() && bytes_read > 0 ) { // =-=-=-=-=-=-=- // debug statistics #ifdef MD5_DEBUG total_bytes_read += bytes_read; #endif // =-=-=-=-=-=-=- // update hasher hasher.update( std::string( buffer, bytes_read ) ); // =-=-=-=-=-=-=- // read some more read_err = fileRead( rsComm, file_obj, buffer, SVR_MD5_BUF_SZ ); if ( read_err.ok() ) { bytes_read = read_err.code(); } else { std::stringstream msg; msg << __FUNCTION__; msg << " - Failed to read buffer from file: \""; msg << fileName; msg << "\""; irods::error result = PASSMSG( msg.str(), read_err ); irods::log( result ); return result.code(); } } // while // =-=-=-=-=-=-=- // close out the file ret = fileClose( rsComm, file_obj ); if ( !ret.ok() ) { irods::error err = PASSMSG( "error on close", ret ); irods::log( err ); } // =-=-=-=-=-=-=- // extract the digest from the hasher object // and copy to outgoing string std::string digest; hasher.digest( digest ); strncpy( chksumStr, digest.c_str(), NAME_LEN ); // =-=-=-=-=-=-=- // debug messaging #ifdef MD5_DEBUG rodsLog( LOG_NOTICE, "fileChksum: chksum = %s, total_bytes_read = %lld", chksumStr, total_bytes_read ); #endif return 0; }
/// =-=-=-=-=-=-=- /// @brief function to query resource for chosen server to which to redirect /// for a given operation error resolve_resource_hierarchy( const std::string& _oper, rsComm_t* _comm, dataObjInp_t* _data_obj_inp, std::string& _out_hier ) { // =-=-=-=-=-=-=- // validate incoming parameters if ( !_comm ) { return ERROR( SYS_INVALID_INPUT_PARAM, "null comm pointer" ); } else if ( !_data_obj_inp ) { return ERROR( SYS_INVALID_INPUT_PARAM, "null data obj inp pointer" ); } // =-=-=-=-=-=-=- // cache the operation, as we may need to modify it std::string oper = _oper; // =-=-=-=-=-=-=- // if this is a put operation then we do not have a first class object resource_ptr resc; file_object_ptr file_obj( new file_object( ) ); // =-=-=-=-=-=-=- // if this is a special collection then we need to get the hier // pass that along and bail as it is not a data object, or if // it is just a not-so-special collection then we continue with // processing the operation, as this may be a create op rodsObjStat_t *rodsObjStatOut = NULL; int spec_stat = collStat( _comm, _data_obj_inp, &rodsObjStatOut ); file_obj->logical_path( _data_obj_inp->objPath ); if ( spec_stat >= 0 ) { if ( rodsObjStatOut->specColl != NULL ) { _out_hier = rodsObjStatOut->specColl->rescHier; freeRodsObjStat( rodsObjStatOut ); return SUCCESS(); } } freeRodsObjStat( rodsObjStatOut ); // =-=-=-=-=-=-=- // extract the resc name keyword from the conditional input char* back_up_resc_name = getValByKey( &_data_obj_inp->condInput, BACKUP_RESC_NAME_KW ); char* dest_resc_name = getValByKey( &_data_obj_inp->condInput, DEST_RESC_NAME_KW ); char* default_resc_name = getValByKey( &_data_obj_inp->condInput, DEF_RESC_NAME_KW ); char* resc_name = getValByKey( &_data_obj_inp->condInput, RESC_NAME_KW ); // =-=-=-=-=-=-=- // assign the keyword in an order, if it applies char* key_word = 0; if ( resc_name ) { key_word = resc_name; } else if ( dest_resc_name ) { key_word = dest_resc_name; } else if ( back_up_resc_name ) { key_word = back_up_resc_name; } // =-=-=-=-=-=-=- // call factory for given obj inp, get a file_object error fac_err = file_object_factory( _comm, _data_obj_inp, file_obj ); // =-=-=-=-=-=-=- // perform an open operation if create is not specificied ( thats all we have for now ) if ( OPEN_OPERATION == oper || WRITE_OPERATION == oper ) { // =-=-=-=-=-=-=- // factory has already been called, test for // success before proceeding if ( !fac_err.ok() ) { std::stringstream msg; msg << "resolve_resource_hierarchy :: failed in file_object_factory"; return PASSMSG( msg.str(), fac_err ); } // =-=-=-=-=-=-=- // consider force flag - we need to consider the default // resc if -f is specified char* force_flag = getValByKey( &_data_obj_inp->condInput, FORCE_FLAG_KW ); if ( force_flag && !key_word ) { key_word = default_resc_name; } // =-=-=-=-=-=-=- // attempt to resolve for an open _out_hier = ""; error ret = resolve_hier_for_open_or_write( _comm, file_obj, key_word, oper, _out_hier ); return ret; } else if ( CREATE_OPERATION == oper ) { // =-=-=-=-=-=-=- // include the default resc name if it applies if ( !key_word && default_resc_name ) { key_word = default_resc_name; } // =-=-=-=-=-=-=- // if we have valid data objects then this could // be actually an open rather than a pure create error ret = SUCCESS(); if ( fac_err.ok() ) { ret = resolve_hier_for_create_or_open( _comm, file_obj, key_word, _data_obj_inp, _out_hier ); } else { // =-=-=-=-=-=-=- // attempt to resolve for a create ret = resolve_hier_for_create( _comm, file_obj, key_word, _data_obj_inp, _out_hier ); } return ret; } // else // =-=-=-=-=-=-=- // should not get here std::stringstream msg; msg << "operation not supported [" << oper << "]"; return ERROR( -1, msg.str() ); } // resolve_resource_hierarchy
int _call_file_modified_for_modification( rsComm_t* rsComm, modDataObjMeta_t* modDataObjMetaInp ) { int status = 0; dataObjInfo_t *dataObjInfo; keyValPair_t *regParam; ruleExecInfo_t rei2; memset( ( char* )&rei2, 0, sizeof( ruleExecInfo_t ) ); rei2.rsComm = rsComm; if ( rsComm != NULL ) { rei2.uoic = &rsComm->clientUser; rei2.uoip = &rsComm->proxyUser; } rei2.doi = modDataObjMetaInp->dataObjInfo; rei2.condInputData = modDataObjMetaInp->regParam; regParam = modDataObjMetaInp->regParam; dataObjInfo = modDataObjMetaInp->dataObjInfo; if ( regParam->len == 0 ) { return ( 0 ); } if ( getValByKey( regParam, ALL_KW ) != NULL ) { /* all copies */ dataObjInfo_t *dataObjInfoHead = NULL; dataObjInfo_t *tmpDataObjInfo; dataObjInp_t dataObjInp; bzero( &dataObjInp, sizeof( dataObjInp ) ); rstrcpy( dataObjInp.objPath, dataObjInfo->objPath, MAX_NAME_LEN ); status = getDataObjInfoIncSpecColl( rsComm, &dataObjInp, &dataObjInfoHead ); if ( status < 0 ) { rodsLog( LOG_NOTICE, "%s - Failed to get data objects.", __FUNCTION__ ); return status; } tmpDataObjInfo = dataObjInfoHead; while ( tmpDataObjInfo != NULL ) { if ( tmpDataObjInfo->specColl != NULL ) { break; } irods::file_object_ptr file_obj( new irods::file_object( rsComm, tmpDataObjInfo ) ); char* pdmo_kw = getValByKey( regParam, IN_PDMO_KW ); if ( pdmo_kw != NULL ) { file_obj->in_pdmo( pdmo_kw ); } irods::error ret = fileModified( rsComm, file_obj ); if ( !ret.ok() ) { std::stringstream msg; msg << __FUNCTION__; msg << " - Failed to signal resource that the data object \""; msg << tmpDataObjInfo->objPath; msg << " was modified."; ret = PASSMSG( msg.str(), ret ); irods::log( ret ); status = ret.code(); } tmpDataObjInfo = tmpDataObjInfo->next; } freeAllDataObjInfo( dataObjInfoHead ); } else { irods::file_object_ptr file_obj( new irods::file_object( rsComm, dataObjInfo ) ); char* pdmo_kw = getValByKey( regParam, IN_PDMO_KW ); if ( pdmo_kw != NULL ) { file_obj->in_pdmo( pdmo_kw ); } irods::error ret = fileModified( rsComm, file_obj ); if ( !ret.ok() ) { std::stringstream msg; msg << __FUNCTION__; msg << " - Failed to signal the resource that the data object \""; msg << dataObjInfo->objPath; msg << "\" was modified."; ret = PASSMSG( msg.str(), ret ); irods::log( ret ); status = ret.code(); } } return status; }