void DataAssigner::set(const mltools::DataConfig &data, int num, bool local) { CHECK_GT(num, 0); parts_.resize(num); if (local) { for (int i = 0; i < num; ++i) { parts_[i].CopyFrom(data); } return; } CHECK_GT(data.replica(), 0); DataConfig files = searchFiles(data); VLOG(1) << "find " << files.file_size() << "files" << "files : " << files.ShortDebugString(); for (int r = 0; r < data.replica(); ++r) { if (data.shuffle()) { shuffleFiles(files); } auto prts = divideFiles(files, num); if (r == 0) { parts_ = prts; } else { for (int i = 0; i < num; ++i) { parts_[i] = prts[i]; } } } VLOG(1) << "divide files into " << num << " parts"; }
QStringList TcFile::searchFiles(const QString& path) { QStringList ret; // 这个函数可以执行任何任务, // 这里只是简单地输出各个文件(夹)的名字 QDir dir(path); QStringList list = dir.entryList(QDir::Dirs, QDir::Name); for ( QStringList::Iterator it=list.begin();it!=list.end();it++ ) { if ( "." != *it && ".." != *it ) { ret.append(searchFiles(path + QDir::separator() + *it)); } } list = dir.entryList(QDir::Files, QDir::Name); for ( QStringList::Iterator it=list.begin();it!=list.end();it++ ) { ret.append(*it); } return ret; }