void DataAssigner::set(const mltools::DataConfig &data, int num, bool local) {
CHECK_GT(num, 0);
parts_.resize(num);
if (local) {
for (int i = 0; i < num; ++i) {
parts_[i].CopyFrom(data);
}
return;
}
CHECK_GT(data.replica(), 0);
DataConfig files = searchFiles(data);
VLOG(1) << "find " << files.file_size() << "files"
<< "files : " << files.ShortDebugString();
for (int r = 0; r < data.replica(); ++r) {
if (data.shuffle()) {
shuffleFiles(files);
}
auto prts = divideFiles(files, num);
if (r == 0) {
parts_ = prts;
} else {
for (int i = 0; i < num; ++i) {
parts_[i] = prts[i];
}
}
}
VLOG(1) << "divide files into " << num << " parts";
}
QStringList TcFile::searchFiles(const QString& path)
{
QStringList ret;
// 这个函数可以执行任何任务，
// 这里只是简单地输出各个文件（夹）的名字
QDir dir(path);
QStringList
list = dir.entryList(QDir::Dirs, QDir::Name);
for ( QStringList::Iterator it=list.begin();it!=list.end();it++ )
{
if ( "." != *it && ".." != *it )
{
ret.append(searchFiles(path + QDir::separator() + *it));
}
}
list = dir.entryList(QDir::Files, QDir::Name);
for ( QStringList::Iterator it=list.begin();it!=list.end();it++ )
{
ret.append(*it);
}
return ret;
}
