void InstancesTest::test_arrange_used_indices()
{
message += "test_arrange_used_indices\n";
Instances i;
Vector<size_t> used_indices;
used_indices = i.arrange_used_indices();
assert_true(used_indices.size() == 0, LOG);
i.set(1);
used_indices = i.arrange_used_indices();
assert_true(used_indices.size() == 1, LOG);
i.set(4);
i.set_use(0, Instances::Training);
i.set_use(1, Instances::Unused);
i.set_use(2, Instances::Testing);
i.set_use(3, Instances::Selection);
used_indices = i.arrange_used_indices();
assert_true(used_indices.size() == 3, LOG);
assert_true(used_indices[0] == 0, LOG);
assert_true(used_indices[1] == 2, LOG);
assert_true(used_indices[2] == 3, LOG);
}
double ThresholdCurve::getROCArea(Instances &tcurve) {
const int n = tcurve.numInstances();
if (RELATION_NAME != tcurve.getRelationName() || (n == 0)) {
return std::numeric_limits<double>::quiet_NaN();
}
const int tpInd = tcurve.attribute(TRUE_POS_NAME).index();
const int fpInd = tcurve.attribute(FALSE_POS_NAME).index();
const double_array tpVals = tcurve.attributeToDoubleArray(tpInd);
const double_array fpVals = tcurve.attributeToDoubleArray(fpInd);
double area = 0.0, cumNeg = 0.0;
const double totalPos = tpVals[0];
const double totalNeg = fpVals[0];
for (int i = 0; i < n; i++) {
double cip, cin;
if (i < n - 1) {
cip = tpVals[i] - tpVals[i + 1];
cin = fpVals[i] - fpVals[i + 1];
}
else {
cip = tpVals[n - 1];
cin = fpVals[n - 1];
}
area += cip * (cumNeg + (0.5 * cin));
cumNeg += cin;
}
area /= (totalNeg * totalPos);
return area;
}
graphmod::Instances from_lines(string file_name, vector<string> stopword_list, unsigned int min_size){
Instances instances;
set<string> stopwords;
for(string w: stopword_list){
stopwords.insert(w);
}
namespace io = boost::iostreams;
ifstream file(file_name.c_str(), ios_base::in);
io::filtering_stream<io::input> in;
in.push(file);
string line;
boost::regex expr("\\s+", boost::regex::perl);
while(in){
getline(in, line);
map<string, vector<string> > instance;
boost::sregex_token_iterator t1(line.begin(), line.end(), expr, -1);
boost::sregex_token_iterator t2;
while(t1 != t2){
string value = *t1;
for(auto& c: value){
c = tolower(c);
}
if(stopwords.count(value) == 0 and value.size() > min_size){
instance["token"].push_back(value);
}
t1++;
}
if(instance["token"].size() > 0){
instances.add(instance);
}
}
return instances;
}
void InstancesTest::test_to_XML(void)
{
message += "test_to_XML\n";
Instances i;
tinyxml2::XMLDocument* document = i.to_XML();
assert_true(document != NULL, LOG);
// Test
i.set(2);
i.set_use(0, Instances::Testing);
i.set_use(1, Instances::Unused);
document = i.to_XML();
i.set();
i.from_XML(*document);
assert_true(i.get_instances_number() == 2, LOG);
assert_true(i.get_use(0) == Instances::Testing, LOG);
assert_true(i.get_use(1) == Instances::Unused, LOG);
}
void InstancesTest::test_get_instances_number(void)
{
message += "test_get_instances_number\n";
Instances i;
assert_true(i.get_instances_number() == 0, LOG);
}
void Instances::copyInstances(const int from, const Instances &dest, const int num)
{
for (int i = 0; i < num; i++)
{
Instance *newInstance = new Instance(dest.instance(i).weight(), dest.instance(i).toDoubleArray());
newInstance->setDataset(const_cast<Instances*>(this));
mInstances.push_back(newInstance);
}
}
void InstancesTest::test_from_XML(void)
{
message += "test_from_XML\n";
Instances i;
tinyxml2::XMLDocument* document = i.to_XML();
i.from_XML(*document);
}
void InstancesTest::test_set(void)
{
message += "test_set\n";
Instances i;
// Instances and inputs and target variables
i.set(1);
assert_true(i.get_instances_number() == 1, LOG);
}
void InstancesTest::test_count_selection_instances_number(void)
{
message += "test_count_selection_instances_number\n";
Instances i;
assert_true(i.count_selection_instances_number() == 0, LOG);
i.set(1);
assert_true(i.count_selection_instances_number() == 0, LOG);
}
int ThresholdCurve::getThresholdInstance(Instances &tcurve, const double threshold) {
if (RELATION_NAME != tcurve.getRelationName() || (tcurve.numInstances() == 0) || (threshold < 0) || (threshold > 1.0)) {
return -1;
}
if (tcurve.numInstances() == 1) {
return 0;
}
double_array tvals = tcurve.attributeToDoubleArray(tcurve.numAttributes() - 1);
int_array sorted = Utils::Sort(tvals);
return binarySearch(sorted, tvals, threshold);
}
void InstancesTest::test_get_display(void)
{
message += "test_get_display\n";
Instances i;
i.set_display(true);
assert_true(i.get_display() == true, LOG);
i.set_display(false);
assert_true(i.get_display() == false, LOG);
}
Licq::ProtocolPlugin::Instances ProtocolPlugin::instances() const
{
Instances list;
Licq::MutexLocker locker(myMutex);
for (std::vector< boost::weak_ptr<PluginInstance> >::const_iterator it =
myInstances.begin(); it != myInstances.end(); ++it)
{
ProtocolPluginInstance::Ptr instance =
boost::dynamic_pointer_cast<ProtocolPluginInstance>(it->lock());
if (instance)
list.push_back(instance);
}
return list;
}
double ThresholdCurve::getNPointPrecision(Instances &tcurve, const int n) {
if (RELATION_NAME != tcurve.getRelationName() || (tcurve.numInstances() == 0)) {
return std::numeric_limits<double>::quiet_NaN();
}
int recallInd = tcurve.attribute(RECALL_NAME).index();
int precisInd = tcurve.attribute(PRECISION_NAME).index();
double_array recallVals = tcurve.attributeToDoubleArray(recallInd);
int_array sorted = Utils::Sort(recallVals);
double isize = 1.0 / (n - 1);
double psum = 0;
for (int i = 0; i < n; i++) {
int pos = binarySearch(sorted, recallVals, i * isize);
double recall = recallVals[sorted[pos]];
double precis = tcurve.instance(sorted[pos]).value(precisInd);
// interpolate figures for non-endpoints
while ((pos != 0) && (pos < sorted.size() - 1)) {
pos++;
double recall2 = recallVals[sorted[pos]];
if (recall2 != recall) {
double precis2 = tcurve.instance(sorted[pos]).value(precisInd);
double slope = (precis2 - precis) / (recall2 - recall);
double offset = precis - recall * slope;
precis = isize * i * slope + offset;
break;
}
}
psum += precis;
}
return psum / n;
}
void InstancesTest::test_arrange_testing_indices(void)
{
message += "test_arrange_testing_indices\n";
Instances i;
Vector<size_t> testing_indices;
testing_indices = i.arrange_testing_indices();
assert_true(testing_indices.size() == 0, LOG);
i.set(1);
testing_indices = i.arrange_testing_indices();
assert_true(testing_indices.size() == 0, LOG);
}
void InstancesTest::test_arrange_generalization_indices(void)
{
message += "test_arrange_generalization_indices\n";
Instances i;
Vector<size_t> generalization_indices;
generalization_indices = i.arrange_generalization_indices();
assert_true(generalization_indices.size() == 0, LOG);
i.set(1);
generalization_indices = i.arrange_generalization_indices();
assert_true(generalization_indices.size() == 0, LOG);
}
void MpiLauncher::getSortedInstances(map<InstanceID,const InstanceDesc*>& sortedInstances,
const Instances& instances,
const boost::shared_ptr<Query>& query)
{
for (Instances::const_iterator i = instances.begin(); i != instances.end(); ++i) {
InstanceID id = i->getInstanceId();
try {
// lid should be equal mpi rank
InstanceID lid = query->mapPhysicalToLogical(id);
sortedInstances[lid] = &(*i);
} catch(SystemException& e) {
if (e.getLongErrorCode() != SCIDB_LE_INSTANCE_OFFLINE) {
throw;
}
}
}
assert(sortedInstances.size() == query->getInstancesCount());
}
void InstancesTest::test_arrange_selection_indices(void)
{
message += "test_arrange_selection_indices\n";
Instances i;
Vector<size_t> selection_indices;
selection_indices = i.arrange_selection_indices();
assert_true(selection_indices.size() == 0, LOG);
i.set(1);
selection_indices = i.arrange_selection_indices();
assert_true(selection_indices.size() == 0, LOG);
}
void InstancesTest::test_count_training_instances_number(void)
{
message += "test_count_training_instances_number\n";
Instances i;
// Test
i.set();
assert_true(i.count_training_instances_number() == 0, LOG);
// Test
i.set(1);
assert_true(i.count_training_instances_number() == 1, LOG);
}
//-----------------------------------------------------------------------
SceneManagerEnumerator::~SceneManagerEnumerator()
{
// Destroy all remaining instances
// Really should have shutdown and unregistered by now, but catch here in case
Instances instancesCopy = mInstances;
for (Instances::iterator i = instancesCopy.begin(); i != instancesCopy.end(); ++i)
{
// destroy instances
for(Factories::iterator f = mFactories.begin(); f != mFactories.end(); ++f)
{
if ((*f)->getMetaData().typeName == i->second->getTypeName())
{
(*f)->destroyInstance(i->second);
mInstances.erase(i->first);
break;
}
}
}
mInstances.clear();
}
void Series::Load3DImage(void* target,
Orthanc::PixelFormat format,
size_t lineStride,
size_t stackStride,
Orthanc::ThreadedCommandProcessor::IListener* listener)
{
using namespace Orthanc;
// Choose the extraction mode, depending on the format of the
// target image.
uint8_t bytesPerPixel;
ImageExtractionMode mode;
switch (format)
{
case PixelFormat_RGB24:
bytesPerPixel = 3;
mode = ImageExtractionMode_Preview;
break;
case PixelFormat_Grayscale8:
bytesPerPixel = 1;
mode = ImageExtractionMode_UInt8; // Preview ???
break;
case PixelFormat_Grayscale16:
bytesPerPixel = 2;
mode = ImageExtractionMode_UInt16;
break;
case PixelFormat_SignedGrayscale16:
bytesPerPixel = 2;
mode = ImageExtractionMode_UInt16;
format = PixelFormat_Grayscale16;
break;
default:
throw OrthancException(ErrorCode_NotImplemented);
}
// Check that the target image is properly sized
unsigned int sx = GetWidth();
unsigned int sy = GetHeight();
if (lineStride < sx * bytesPerPixel ||
stackStride < sx * sy * bytesPerPixel)
{
throw OrthancException(ErrorCode_BadRequest);
}
if (sx == 0 || sy == 0 || GetInstanceCount() == 0)
{
// Empty image, nothing to do
if (listener)
listener->SignalSuccess(0);
return;
}
/**
* Order the stacks according to their distance along the slice
* normal (using the "Image Position Patient" tag). This works
* even if the "SliceLocation" tag is absent.
**/
SliceLocator locator(GetInstance(0));
typedef std::map<float, Instance*> Instances;
Instances instances;
for (unsigned int i = 0; i < GetInstanceCount(); i++)
{
float dist = locator.ComputeSliceLocation(GetInstance(i));
instances[dist] = &GetInstance(i);
}
if (instances.size() != GetInstanceCount())
{
// Several instances have the same Z coordinate
throw OrthancException(ErrorCode_NotImplemented);
}
// Submit the download of each stack as a set of commands
ThreadedCommandProcessor processor(connection_.GetThreadCount());
if (listener != NULL)
{
processor.SetListener(*listener);
}
uint8_t* stackTarget = reinterpret_cast<uint8_t*>(target);
for (Instances::iterator it = instances.begin(); it != instances.end(); it++)
{
processor.Post(new ImageDownloadCommand(*it->second, format, mode, stackTarget, lineStride));
stackTarget += stackStride;
}
// Wait for all the stacks to be downloaded
if (!processor.Join())
{
throw OrthancException(ErrorCode_NetworkProtocol);
}
}
static void AnonymizeOrModifyResource(DicomModification& modification,
MetadataType metadataType,
ChangeType changeType,
ResourceType resourceType,
RestApiPostCall& call)
{
bool isFirst = true;
Json::Value result(Json::objectValue);
ServerContext& context = OrthancRestApi::GetContext(call);
typedef std::list<std::string> Instances;
Instances instances;
std::string id = call.GetUriComponent("id", "");
context.GetIndex().GetChildInstances(instances, id);
if (instances.empty())
{
return;
}
/**
* Loop over all the instances of the resource.
**/
for (Instances::const_iterator it = instances.begin();
it != instances.end(); ++it)
{
LOG(INFO) << "Modifying instance " << *it;
std::auto_ptr<ServerContext::DicomCacheLocker> locker;
try
{
locker.reset(new ServerContext::DicomCacheLocker(OrthancRestApi::GetContext(call), *it));
}
catch (OrthancException&)
{
// This child instance has been removed in between
continue;
}
ParsedDicomFile& original = locker->GetDicom();
DicomInstanceHasher originalHasher = original.GetHasher();
/**
* Compute the resulting DICOM instance.
**/
std::auto_ptr<ParsedDicomFile> modified(original.Clone());
modification.Apply(*modified);
DicomInstanceToStore toStore;
toStore.SetParsedDicomFile(*modified);
/**
* Prepare the metadata information to associate with the
* resulting DICOM instance (AnonymizedFrom/ModifiedFrom).
**/
DicomInstanceHasher modifiedHasher = modified->GetHasher();
if (originalHasher.HashSeries() != modifiedHasher.HashSeries())
{
toStore.AddMetadata(ResourceType_Series, metadataType, originalHasher.HashSeries());
}
if (originalHasher.HashStudy() != modifiedHasher.HashStudy())
{
toStore.AddMetadata(ResourceType_Study, metadataType, originalHasher.HashStudy());
}
if (originalHasher.HashPatient() != modifiedHasher.HashPatient())
{
toStore.AddMetadata(ResourceType_Patient, metadataType, originalHasher.HashPatient());
}
assert(*it == originalHasher.HashInstance());
toStore.AddMetadata(ResourceType_Instance, metadataType, *it);
/**
* Store the resulting DICOM instance into the Orthanc store.
**/
std::string modifiedInstance;
if (context.Store(modifiedInstance, toStore) != StoreStatus_Success)
{
LOG(ERROR) << "Error while storing a modified instance " << *it;
return;
}
// Sanity checks in debug mode
assert(modifiedInstance == modifiedHasher.HashInstance());
/**
* Compute the JSON object that is returned by the REST call.
**/
if (isFirst)
{
std::string newId;
switch (resourceType)
{
case ResourceType_Series:
newId = modifiedHasher.HashSeries();
break;
case ResourceType_Study:
newId = modifiedHasher.HashStudy();
break;
case ResourceType_Patient:
newId = modifiedHasher.HashPatient();
break;
default:
throw OrthancException(ErrorCode_InternalError);
}
result["Type"] = EnumerationToString(resourceType);
result["ID"] = newId;
result["Path"] = GetBasePath(resourceType, newId);
result["PatientID"] = modifiedHasher.HashPatient();
isFirst = false;
}
}
call.GetOutput().AnswerJson(result);
}
graphmod::Instances from_conll(vector<string> file_names, vector<string> _keep_verbs, int window, unsigned int limit=0, unsigned int per_verb_limit=0){
Instances instances;
set<string> keep_verbs;
for(string verb: _keep_verbs){
keep_verbs.insert(verb);
}
std::map<std::string, unsigned int> verb_counts;
ConllLoader sentences(file_names);
//boost::regex expr("\\n\\s*\\n", boost::regex::perl);
//boost::regex_iterator
//int total = 0;
while(not sentences.eof() and (limit == 0 or instances.size() < limit)){
auto cs = sentences.next();
for(ConllWord cw: cs){
if(cw.get_fine_tag()[0] == 'V' and (keep_verbs.count(cw.get_lemma()) > 0 or keep_verbs.size() == 0)){
map<string, vector<string> > instance;
int verb_index = cw.get_index();
instance["verb"] = {cw.get_lemma()};
if(per_verb_limit > 0 and verb_counts[cw.get_lemma()] > per_verb_limit){
continue;
}
verb_counts[cw.get_lemma()]++;
instance["verb_tag"] = {cw.get_fine_tag()};
instance["tag"].resize(0);
instance["gr"].resize(0);
instance["lemma"].resize(0);
if(window > 0){
for(ConllWord ow: cs.get_near(cw, window)){
string tag = ow.get_fine_tag();
instance["tag"].push_back(ow.get_fine_tag());
instance["lemma"].push_back(ow.get_lemma());
stringstream ss;
ss << ow.get_index() - verb_index;
instance["gr"].push_back(ss.str());
}
}
else{
//instance["relation"].resize(0);
for(ConllWord ow: cs.get_related(cw)){
stringstream ss;
string tag = ow.get_fine_tag(), gr = ow.get_relation();
if(function_tag(tag[0]) == true){
ss << gr << "(" << tag << "-" << ow.get_lemma() << "," << cw.get_fine_tag() << ")";
}
else{
ss << gr << "(" << tag << "," << cw.get_fine_tag() << ")";
}
instance["tag"].push_back(tag);
//instance["gr"].push_back(ss.str());
instance["gr"].push_back(gr);
instance["lemma"].push_back(ow.get_lemma());
}
}
instances.add(instance);
}
}
}
return instances;
//cout << total << endl;
}
Instances *ThresholdCurve::getCurve(std::vector<Prediction*> predictions, const int classIndex) {
if ((predictions.size() == 0) || ((static_cast<NominalPrediction*>(predictions.at(0)))->distribution().size() <= classIndex)) {
return nullptr;
}
double totPos = 0, totNeg = 0;
double_array probs = getProbabilities(predictions, classIndex);
// Get distribution of positive/negatives
for (int i = 0; i < probs.size(); i++) {
NominalPrediction *pred = static_cast<NominalPrediction*>(predictions.at(i));
if (pred->actual() == Prediction::MISSING_VALUE) {
std::cout << " Skipping prediction with missing class value";
continue;
}
if (pred->weight() < 0) {
std::cout << " Skipping prediction with negative weight";
continue;
}
if (pred->actual() == classIndex) {
totPos += pred->weight();
}
else {
totNeg += pred->weight();
}
}
Instances *insts = makeHeader();
int_array sorted = Utils::Sort(probs);
TwoClassStats *tc = new TwoClassStats(totPos, totNeg, 0, 0);
double threshold = 0;
double cumulativePos = 0;
double cumulativeNeg = 0;
for (int i = 0; i < sorted.size(); i++) {
if ((i == 0) || (probs[sorted[i]] > threshold)) {
tc->setTruePositive(tc->getTruePositive() - cumulativePos);
tc->setFalseNegative(tc->getFalseNegative() + cumulativePos);
tc->setFalsePositive(tc->getFalsePositive() - cumulativeNeg);
tc->setTrueNegative(tc->getTrueNegative() + cumulativeNeg);
threshold = probs[sorted[i]];
insts->add(*makeInstance(tc, threshold));
cumulativePos = 0;
cumulativeNeg = 0;
if (i == sorted.size() - 1) {
break;
}
}
NominalPrediction *pred = static_cast<NominalPrediction*>(predictions.at(sorted[i]));
if (pred->actual() == Prediction::MISSING_VALUE) {
std::cout << " Skipping prediction with missing class value";
continue;
}
if (pred->weight() < 0) {
std::cout << " Skipping prediction with negative weight";
continue;
}
if (pred->actual() == classIndex) {
cumulativePos += pred->weight();
}
else {
cumulativeNeg += pred->weight();
}
}
// make sure a zero point gets into the curve
if (tc->getFalseNegative() != totPos || tc->getTrueNegative() != totNeg) {
tc = new TwoClassStats(0, 0, totNeg, totPos);
threshold = probs[sorted[sorted.size() - 1]] + 10e-6;
insts->add(*makeInstance(tc, threshold));
}
return insts;
}
