const void *getResult(const void *firstRow)
{
IHThorAggregateArg *helper = (IHThorAggregateArg *)baseHelper.get();
unsigned numPartialResults = container.queryJob().querySlaves();
if (1 == numPartialResults)
return firstRow;
CThorExpandingRowArray partialResults(*this, this, true, stableSort_none, true, numPartialResults);
if (hadElement)
partialResults.setRow(0, firstRow);
--numPartialResults;
size32_t sz;
while (numPartialResults--)
{
CMessageBuffer msg;
rank_t sender;
if (!receiveMsg(msg, RANK_ALL, mpTag, &sender))
return NULL;
if (abortSoon)
return NULL;
msg.read(sz);
if (sz)
{
assertex(NULL == partialResults.query(sender-1));
CThorStreamDeserializerSource mds(sz, msg.readDirect(sz));
RtlDynamicRowBuilder rowBuilder(queryRowAllocator());
size32_t sz = queryRowDeserializer()->deserialize(rowBuilder, mds);
partialResults.setRow(sender-1, rowBuilder.finalizeRowClear(sz));
}
}
RtlDynamicRowBuilder rowBuilder(queryRowAllocator(), false);
bool first = true;
numPartialResults = container.queryJob().querySlaves();
unsigned p=0;
for (;p<numPartialResults; p++)
{
const void *row = partialResults.query(p);
if (row)
{
if (first)
{
first = false;
sz = cloneRow(rowBuilder, row, queryRowMetaData());
}
else
sz = helper->mergeAggregate(rowBuilder, row);
}
}
if (first)
sz = helper->clearAggregate(rowBuilder);
return rowBuilder.finalizeRowClear(sz);
}
Common &Common::Carrier::common() const
{
if (!mCommon) {
const CssmDbRecordAttributeData &attrs
= dynamic_cast<const Record *>(this)->attributes();
RefPointer<Common> rpc;
rpc = Table<Common>(mds()).fetch(
Attribute("ModuleID") == string(attrs[0]),
CSSMERR_DL_ENDOFDATA);
mCommon = rpc;
}
return *mCommon;
}
void Network::requestConfiguration()
{
if (_id == ZT_TEST_NETWORK_ID) // pseudo-network-ID, uses locally generated static config
return;
if (controller() == RR->identity.address()) {
if (RR->localNetworkController) {
SharedPtr<NetworkConfig> nconf(config2());
Dictionary newconf;
switch(RR->localNetworkController->doNetworkConfigRequest(InetAddress(),RR->identity,RR->identity,_id,Dictionary(),newconf)) {
case NetworkController::NETCONF_QUERY_OK:
this->setConfiguration(newconf,true);
return;
case NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND:
this->setNotFound();
return;
case NetworkController::NETCONF_QUERY_ACCESS_DENIED:
this->setAccessDenied();
return;
default:
return;
}
} else {
this->setNotFound();
return;
}
}
TRACE("requesting netconf for network %.16llx from controller %s",(unsigned long long)_id,controller().toString().c_str());
// TODO: in the future we will include things like join tokens here, etc.
Dictionary metaData;
metaData.setHex(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,ZEROTIER_ONE_VERSION_MAJOR);
metaData.setHex(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,ZEROTIER_ONE_VERSION_MINOR);
metaData.setHex(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,ZEROTIER_ONE_VERSION_REVISION);
std::string mds(metaData.toString());
Packet outp(controller(),RR->identity.address(),Packet::VERB_NETWORK_CONFIG_REQUEST);
outp.append((uint64_t)_id);
outp.append((uint16_t)mds.length());
outp.append((const void *)mds.data(),(unsigned int)mds.length());
{
Mutex::Lock _l(_lock);
if (_config)
outp.append((uint64_t)_config->revision());
else outp.append((uint64_t)0);
}
RR->sw->send(outp,true,0);
}
TEST_F(MetaDataStoreTest, path_and_id)
{
vfs::MetaDataStore mds(std::static_pointer_cast<yt::LockedArakoon>(registry_),
cluster_id_,
vfs::UseCache::T);
const vfs::FrontendPath dpath("/dir");
const vfs::Permissions pms(S_IWUSR bitor S_IRUSR);
vfs::DirectoryEntryPtr
dentry(boost::make_shared<vfs::DirectoryEntry>(vfs::DirectoryEntry::Type::Directory,
mds.alloc_inode(),
pms,
vfs::UserId(::getuid()),
vfs::GroupId(::getgid())));
mds.add(dpath,
dentry);
ASSERT_EQ(dpath,
mds.find_path(dentry->object_id()));
{
const vfs::DirectoryEntryPtr p(mds.find(dentry->object_id()));
ASSERT_TRUE(p != nullptr);
ASSERT_TRUE(*dentry == *p);
}
const vfs::FrontendPath fpath(dpath / "file");
vfs::DirectoryEntryPtr
fentry(boost::make_shared<vfs::DirectoryEntry>(vfs::DirectoryEntry::Type::File,
mds.alloc_inode(),
pms,
vfs::UserId(::getuid()),
vfs::GroupId(::getgid())));
mds.add(dentry->object_id(),
fpath.filename().string(),
fentry);
ASSERT_EQ(fpath,
mds.find_path(fentry->object_id()));
{
const vfs::DirectoryEntryPtr p(mds.find(fentry->object_id()));
ASSERT_TRUE(p != nullptr);
ASSERT_TRUE(*fentry == *p);
}
}
TEST_F(MetaDataStoreTest, root)
{
vfs::MetaDataStore mds(std::static_pointer_cast<yt::LockedArakoon>(registry_),
cluster_id_,
vfs::UseCache::F);
const vfs::FrontendPath root("/");
vfs::DirectoryEntryPtr dentry(mds.find(root));
ASSERT_TRUE(dentry != nullptr);
EXPECT_EQ(root,
mds.find_path(dentry->object_id()));
vfs::DirectoryEntryPtr dentry2(mds.find(dentry->object_id()));
ASSERT_TRUE(dentry2 != nullptr);
}
virtual void process()
{
CMasterActivity::process();
bool results = false;
unsigned nslaves = container.queryJob().querySlaves();
while (nslaves--)
{
CMessageBuffer mb;
if (abortSoon || !receiveMsg(mb, RANK_ALL, replyTag, NULL)) break;
StringBuffer str;
mb.getSender().getUrlStr(str);
size32_t sz;
mb.read(sz);
if (sz)
{
if (results)
throw MakeThorException(TE_UnexpectedMultipleSlaveResults, "Received greater than one result from slaves");
IHThorRemoteResultArg *helper = (IHThorRemoteResultArg *)queryHelper();
Owned<IThorRowInterfaces> resultRowIf = createRowInterfaces(helper->queryOutputMeta());
CThorStreamDeserializerSource mds(sz, mb.readDirect(sz));
RtlDynamicRowBuilder rowBuilder(resultRowIf->queryRowAllocator());
size32_t sz = resultRowIf->queryRowDeserializer()->deserialize(rowBuilder, mds);
OwnedConstThorRow result = rowBuilder.finalizeRowClear(sz);
helper->sendResult(result);
results = true;
}
}
if (!results && !abortSoon)
{
ActPrintLog("WARNING: no results");
IHThorRemoteResultArg *helper = (IHThorRemoteResultArg *)queryHelper();
//helper->sendResult(NULL);
// Jake I think this always cores (so raise exception instead)
throw MakeThorException(TE_UnexpectedMultipleSlaveResults, "Received no results from slaves");
}
}
void Host::CopyModuleAppResources(std::string& modulePath)
{
this->logger->Trace("CopyModuleAppResources: %s", modulePath.c_str());
std::string appDir = this->application->path;
Path appPath(this->application->path);
try
{
Path moduleDir(modulePath);
moduleDir = moduleDir.parent();
std::string mds(moduleDir.toString());
const char* platform = this->GetPlatform();
std::string resources_dir = FileUtils::Join(mds.c_str(), "AppResources", NULL);
std::string plt_resources_dir = FileUtils::Join(resources_dir.c_str(), platform, NULL);
std::string all_resources_dir = FileUtils::Join(resources_dir.c_str(), "all", NULL);
File platformAppResourcesDir(plt_resources_dir);
File allAppResourcesDir(all_resources_dir);
if (platformAppResourcesDir.exists()
&& platformAppResourcesDir.isDirectory())
{
std::vector<File> files;
platformAppResourcesDir.list(files);
for (size_t i = 0; i < files.size(); i++)
{
File f = files.at(i);
Path targetPath(appPath, Path(Path(f.path()).getBaseName()));
File targetFile(targetPath);
this->logger->Trace("target: %s", targetFile.path().c_str());
if (!targetFile.exists())
{
this->logger->Trace("Copying : %s to %s", f.path().c_str(), appDir.c_str());
f.copyTo(appDir);
}
else
{
this->logger->Trace("SKIP Copying : %s to %s", f.path().c_str(), appDir.c_str());
}
}
}
if (allAppResourcesDir.exists()
&& allAppResourcesDir.isDirectory())
{
std::vector<File> files;
allAppResourcesDir.list(files);
for (size_t i = 0; i < files.size(); i++)
{
File f = files.at(i);
Path targetPath(appPath, Path(Path(f.path()).getBaseName()));
File targetFile(targetPath);
this->logger->Trace("target: %s", targetFile.path().c_str());
if (!targetFile.exists())
{
this->logger->Trace("Copying: %s to %s", f.path().c_str(), appDir.c_str());
f.copyTo(appDir);
}
else
{
this->logger->Trace("SKIP Copying : %s to %s", f.path().c_str(), appDir.c_str());
}
}
}
}
catch (Poco::Exception &exc)
{
// Handle..
}
}
// Use boost::program_options to parse command line input
// This does pretty much all the parameter checking needed
int parseCommandLine (int argc, char *argv[], po::variables_map& vm)
{
int failed = 0;
//Required options
po::options_description required("Required options");
required.add_options()
("kmers,k", po::value<std::string>(), "dsm kmer output file (not needed if using --mds_concat)")
("pheno,p", po::value<std::string>(), ".pheno metadata");
po::options_description mds("MDS options");
mds.add_options()
("output,o", po::value<std::string>(), "output prefix for new dsm file")
("no_mds", "do not perform MDS; output subsampled matrix instead")
("write_distances", "write csv of distance matrix")
("mds_concat", po::value<std::string>(), "list of subsampled matrices to use in MDS. Performs only MDS; implies --no_filtering")
("pc", po::value<int>()->default_value(pc_default), "number of principal coordinates to output")
("size", po::value<long int>()->default_value(size_default), "number of kmers to use in MDS")
("threads", po::value<int>()->default_value(1), ("number of threads. Suggested: " + std::to_string(std::thread::hardware_concurrency())).c_str());
//Optional filtering parameters
//NB pval cutoffs are strings for display, and are converted to floats later
po::options_description filtering("Filtering options");
filtering.add_options()
("no_filtering", "turn off all filtering and do not output new kmer file")
("max_length", po::value<long int>()->default_value(max_length_default), "maximum kmer length")
("maf", po::value<double>()->default_value(maf_default), "minimum kmer frequency")
("min_words", po::value<int>(), "minimum kmer occurences. Overrides --maf");
po::options_description other("Other options");
other.add_options()
("version", "prints version and exits")
("help,h", "full help message");
po::options_description all;
all.add(required).add(mds).add(filtering).add(other);
try
{
po::store(po::command_line_parser(argc, argv).options(all).run(), vm);
if (vm.count("help"))
{
printHelp(all);
failed = 1;
}
else if (vm.count("version"))
{
std::cout << VERSION << std::endl;
failed = 1;
}
else
{
po::notify(vm);
failed = 0;
// Check input files exist, and can stat
if ((vm.count("kmers") && !fileStat(vm["kmers"].as<std::string>())) || (vm.count("pheno") && !fileStat(vm["pheno"].as<std::string>())))
{
failed = 1;
}
}
}
catch (po::error& e)
{
// Report errors from boost library
std::cerr << "Error in command line input: " << e.what() << "\n";
std::cerr << "Run 'kmds --help' for full option listing\n\n";
std::cerr << required << "\n" << other << "\n";
failed = 1;
}
return failed;
}
ADSRangeLastAfter::ADSRangeLastAfter( EnvisatFile & envfile,
int ads_idx , int mds_idx, const TimeDelta & line_interval )
{
int idx ;
TimeDelta t_mds , t_ads , t_ads_prev ;
/* abs.time tolerance */
TimeDelta atol = line_interval * 0.5 ;
/* MDS and ADS descriptor classes */
DataSet mds( envfile, mds_idx ) ;
DataSet ads( envfile, ads_idx ) ;
/* read the times of the first and last measurements */
mjd_m_first = mds.getMJD(0) ; /* MDJ time of the first MDS record */
mjd_m_last = mds.getMJD(-1) ; /* MDJ time of the last MDS record */
/* look-up the the first applicable ADSR */
idx = 0 ;
t_mds = mjd_m_first + atol ; /*time of the first MDSR + tolerance */
t_ads = ads.getMJD(0) ; /*time of the first ADSR */
t_ads_prev = t_ads ; /* holds previous ADSR */
if ( t_ads < t_mds )
{
for( idx = 1 ; idx < ads.nrec ; ++idx )
{
t_ads = ads.getMJD(idx) ;
if ( t_ads >= t_mds ) break ;
t_ads_prev = t_ads ;
}
}
/* store the first applicable ASDR */
idx_first = idx - 1 ; /* sets -1 if no match */
mjd_first = t_ads_prev ; /* set time of the first rec. if no match */
/* look-up the the last applicable ADSR */
idx = ads.nrec-2 ;
t_mds = mjd_m_last - atol ; /* time of the last MDSR - tolerance */
t_ads = ads.getMJD(-1) ; /* time of the first ADSR */
t_ads_prev = t_ads ; /* holds previous ADSR */
if ( t_ads > t_mds )
{
for( idx = ads.nrec-2 ; idx >= 0 ; --idx )
{
t_ads = ads.getMJD(idx) ;
if ( t_ads <= t_mds ) break ;
t_ads_prev = t_ads ;
}
}
/* store the last applicable ASDR */
idx_last = idx + 1 ; /* sets ads.nrec if no match */
mjd_last = t_ads_prev ; /* set time of the last rec. if no match */
/* valuate the line offsets */
off_first = (int)floor( 0.5 + ( mjd_m_first - mjd_first ) / line_interval ) ;
off_last = (int)floor( 0.5 + ( mjd_last - mjd_m_last ) / line_interval ) ;
} ;
// make a dir, auto make dirs if parent not exist
const bool mds(const char* p) {
path parent(path(p).dirname());
if (parent.isndir() AND !mds(parent.tocstr())) return false;
return md(p);
}
