ParameterMap parseOptions(int argc, char* argv[]) {
if (argc < 3) {
usage();
}
QStringList args;
for (int i=0; i<argc; i++) args << argv[i];
ParameterMap options;
options.insert("inputFiles", args[1]);
options.insert("dataset", args[2]);
// this is ugly, but works...
for (int i=0; i<args.size(); i++) {
if (args[i].startsWith("--select=")) {
descs_select = args[i].mid(9).split(",");
args.removeAt(i);
i--;
}
if (args[i].startsWith("--exclude=")) {
descs_exclude = args[i].mid(10).split(",");
args.removeAt(i);
i--;
}
if (args[i].startsWith("--reflayout=")) {
layout = new PointLayout();
Point refp; refp.load(args[i].mid(12));
*layout = refp.layout();
args.removeAt(i);
i--;
}
if (args[i].startsWith("--nthreads=")) {
QThreadPool::globalInstance()->setMaxThreadCount(args[i].mid(11).toInt());
args.removeAt(i);
i--;
}
}
int start = 0, end = 10000000;
if (args.size() > 3) start = args[3].toInt();
if (args.size() > 4) end = args[4].toInt();
if (args.size() > 5) usage();
options.insert("start", start);
options.insert("end", end);
return options;
}
Transformation Normalize::analyze(const DataSet* dataset, const Region& descs) const {
G_INFO("Doing normalize analysis...");
checkDataSet(dataset);
checkMinPoints(dataset, 2);
Region region = checkFixedLength(descs.select(RealType), dataset->layout());
// get statistics on the input dataset
Point mmin, mmax, mean, var;
if (!_params.contains("outliers") || _params.value("outliers").toDouble() < 0.0) {
getStats(*dataset, region, mmin, mmax, mean, var);
}
else {
getStatsWithOutliers(*dataset, region, mmin, mmax, mean, var);
}
Transformation result(dataset->layout());
result.analyzerName = "normalize";
result.analyzerParams = _params;
result.applierName = "dotproduct";
// insert info values inside Transfo object: mean & var
ParameterMap stats;
foreach (const Segment& seg, region.segments) {
// FIXME: not necessarily correct if some adjacent descriptors have been
// merged into a single segment
QString descriptorName = mmin.layout().descriptorName(RealType, FixedLength, seg.begin);
ParameterMap meanvar;
meanvar.insert("mean", mean.value(descriptorName));
meanvar.insert("var", var.value(descriptorName));
stats.insert(descriptorName, meanvar);
}
result.info = stats;
// insert actual Transformation parameters
ParameterMap& applierParams = result.params;
applierParams.insert("applyClipping", _params.value("applyClipping", false));
ParameterMap coeffMap;
foreach (const Segment& seg, region.segments) {
QString descriptorName = mmin.layout().descriptorName(RealType, FixedLength, seg.begin);
coeffMap.insert(descriptorName, getNormalizationCoeffs(descriptorName, mmin, mmax, mean, var));
}
applierParams.insert("coeffs", coeffMap);
G_DEBUG(GAlgorithms, "normalization success!");
return result;
}
ReceiveMessageResponse*
SQSConnection::receiveMessage (const std::string &aQueueUrl,
int aNumberOfMessages,
int aVisibilityTimeout,
bool aDecode) {
ParameterMap lMap;
if (aNumberOfMessages != 0) {
std::stringstream s;
s << aNumberOfMessages;
lMap.insert (ParameterPair ("MaxNumberOfMessages", s.str()));
}
if (aVisibilityTimeout > -1) {
std::stringstream s;
s << aVisibilityTimeout;
lMap.insert (ParameterPair ("VisibilityTimeout", s.str()));
}
return receiveMessage (aQueueUrl, lMap, aDecode);
}
CreateQueueResponse*
SQSConnection::createQueue ( const std::string &aQueueName, int aDefaultVisibilityTimeout)
{
ParameterMap lMap;
lMap.insert ( ParameterPair ( "QueueName", aQueueName ) );
if (aDefaultVisibilityTimeout > -1 ) {
std::stringstream s;
s << aDefaultVisibilityTimeout;
lMap.insert ( ParameterPair ( "DefaultVisibilityTimeout", s.str() ) );
}
CreateQueueHandler lHandler;
makeQueryRequest ( "CreateQueue", &lMap, &lHandler );
if (lHandler.isSuccessful()) {
setCommons(lHandler, lHandler.theCreateQueueResponse);
return lHandler.theCreateQueueResponse;
} else {
throw CreateQueueException( lHandler.getQueryErrorResponse() );
}
}
ParameterMap parseOptions(int argc, char* argv[]) {
if (argc < 2) {
G_ERROR("not enough arguments...\n\n"
"Usage: " << argv[0] << " [-v] dataset_file");
}
ParameterMap options;
options.insert("verbose", false);
if (argc >= 3) {
if (QString(argv[1]) == "-v") {
options.insert("verbose", true);
options.insert("dataset", argv[2]);
}
else if (QString(argv[2]) == "-v") {
options.insert("verbose", true);
options.insert("dataset", argv[1]);
}
}
else {
options.insert("dataset", argv[1]);
}
return options;
}
ParameterMap Driver::parameters() {
if(!isValid()) {
return ParameterMap();
}
ParameterMap parameterMap;
const JSList *parameters = jackctl_driver_get_parameters(_jackDriver);
while(parameters) {
Parameter p = Parameter((jackctl_parameter_t*)parameters->data);
parameterMap.insert(p.name(), p);
parameters = parameters->next;
}
return parameterMap;
}
DeleteMessageResponse*
SQSConnection::deleteMessage(const std::string &aQueueUrl, const std::string &aReceiptHandle)
{
ParameterMap lMap;
lMap.insert ( ParameterPair ( "ReceiptHandle", aReceiptHandle ) );
DeleteMessageHandler lHandler;
makeQueryRequest ( aQueueUrl, "DeleteMessage", &lMap, &lHandler );
if (lHandler.isSuccessful()) {
setCommons(lHandler, lHandler.theDeleteMessageResponse);
return lHandler.theDeleteMessageResponse;
} else {
throw DeleteMessageException( lHandler.getQueryErrorResponse() );
}
}
ListQueuesResponse*
SQSConnection::listQueues(const std::string &aQueueNamePrefix)
{
ParameterMap lMap;
if (aQueueNamePrefix != "") {
lMap.insert ( ParameterPair ( "QueueNamePrefix", aQueueNamePrefix ) );
}
ListQueuesHandler lHandler;
makeQueryRequest ( "ListQueues", &lMap, &lHandler );
if (lHandler.isSuccessful()) {
setCommons(lHandler, lHandler.theListQueuesResponse);
return lHandler.theListQueuesResponse;
} else {
throw ListQueuesException( lHandler.getQueryErrorResponse() );
}
}
SendMessageResponse*
SQSConnection::sendMessage(const std::string &aQueueUrl, const std::string &aMessageBody, bool aEncode)
{
ParameterMap lMap;
long lBody64Len;
std::string enc;
if (aEncode)
enc = AWSConnection::base64Encode(aMessageBody.c_str(), aMessageBody.size(), lBody64Len);
else
enc = aMessageBody;
if (enc.size() > 32768) {
std::stringstream lTmp;
lTmp << "Message larger than 32kB : " << enc.size() / 1024 << " kb";
throw SendMessageException( QueryErrorResponse("1", lTmp.str(), "", "") );
}
lMap.insert ( ParameterPair ( "MessageBody", enc ) );
return sendMessage(aQueueUrl, lMap);
}
ParameterMap Normalize::getNormalizationCoeffs(const QString& descriptorName,
const Point& mmin, const Point& mmax,
const Point& mean, const Point& var) const {
RealDescriptor a, b;
if (!_params.contains("type") || // default case: type was not defined
_params.value("type") == "range") {
// normalization using minimum and maximum
// y = ax+b, y = (x - min)/(max - min), a=1/(max-min) and b = -min/(max-min)
// normalization as a vector (default)
if (!_params.contains("independent") || // default case
!_params.value("independent").toBool()) {
RealDescriptor maxd = mmax.value(descriptorName);
RealDescriptor mind = mmin.value(descriptorName);
Real maxv = maxd[0], minv = mind[0];
for (int i=1; i<maxd.size(); i++) maxv = std::max(maxv, maxd[i]);
for (int i=1; i<mind.size(); i++) minv = std::min(minv, mind[i]);
a = RealDescriptor(maxd.size(), 1 / (maxv-minv) );
b = RealDescriptor(mind.size(), -minv / (maxv - minv) );
}
// normalization each dimension separately
else {
a = (mmax.value(descriptorName) - mmin.value(descriptorName)).inverse();
b = -mmin.value(descriptorName) * a;
}
}
else if (_params.value("type") == "variance") {
// normalization using 4 * stddev, between 0 and 1 (centered around 0.5)
// y = ax+b, y = (x - mean)/(4*stddev) + 0.5, thus
// a=1/(4*stddev) and b = -mean/(4*stddev)+0.5
// normalization as a vector (default)
if (!_params.contains("independent") || // default case
!_params.value("independent").toBool()) {
throw GaiaException("This type of normalization (type=variance, independent=false)"
" is not implemented at the moment");
}
// normalization each dimension separately
else {
a = Real(1.0) / (Real(4.0) * sqrt(var.value(descriptorName)));
b = -mean.value(descriptorName) * a + 0.5;
}
}
else {
throw GaiaException("Normalize: unknown normalization type: ", _params.value("type").toString());
}
//G_DEBUG(GAlgorithms, "a:" << a << "b:" << b);
// SECURITY CHECK - shouldn't happen, apply cleaner transfo before if it does...
// happens for 2D arrays, where 2 first values are dimensions, hence constant, but
// as a vector doesn't get cleaned
if (a.isinf() || a.isnan()) {
throw GaiaException("ERROR: Division by zero in ", descriptorName);
}
ParameterMap coeffs;
coeffs.insert("a", a);
coeffs.insert("b", b);
return coeffs;
}
void addParameter( SLSF::Parameter parameter ) {
_parameterMap.insert( std::make_pair( static_cast< std::string >( parameter.name() ), parameter ) );
}
int main(int argc, char* argv[]) {
QCoreApplication app(argc, argv);
ParameterMap options = parseOptions(argc, argv);
gaia2::init();
gaia2::verbose = true;
if (options.value("verbose").toBool()) {
setDebugLevel(GTools);
}
DataSet dataset;
int npoints;
int dimension;
try {
G_INFO("Loading dataset: " << options.value("dataset"));
dataset.load(options.value("dataset"));
npoints = dataset.size();
dimension = dataset.layout().correspondingRegion().dimension();
}
catch (GaiaException& e) {
G_ERROR(e.msg());
exit(1);
}
G_INFO("DataSet name: " << dataset.name());
G_INFO("Total number of points: " << npoints);
G_INFO("Dimension: " << dimension);
G_INFO("");
// ---------------------------------------------------------------------------
G_INFO("Copying data into SuperMatrix (naive FrozenDataset)...");
G_INFO("fixed real data dimension = " << dataset[0]->frealData().size());
G_INFO("(note: should be 175 for gaiabench)");
G_INFO("");
float* frozenDataSet = new float[npoints*dimension];
for (int i=0; i<npoints; i++) {
const int offset = i*dimension;
const RealDescriptor& d = dataset[i]->frealData();
for (int j=0; j<dimension; j++) {
frozenDataSet[offset+j] = d[j];
}
}
float* frozenDataSet2 = new float[npoints*25];
for (int i=0; i<npoints; i++) {
const int offset = i*25;
const RealDescriptor& d = dataset[i]->frealData();
for (int j=0; j<25; j++) {
frozenDataSet2[offset+j] = d[150+j];
}
}
G_INFO("Copying data into gaia::FrozenDataSet...");
FrozenDataSet frozenDS;
frozenDS.fromDataSet(dataset);
gaia2::verbose = false;
ParameterMap eucp;
eucp.insert("descriptorNames", "pca");
DataSet* eucds = transform(&dataset, "select", eucp);
FrozenDataSet frozeuc;
frozeuc.fromDataSet(*eucds);
delete eucds;
gaia2::verbose = true;
G_INFO("FrozenDataSets ready");
G_INFO("");
// ---------------------------------------------------------------------------
G_INFO(G_HLINE "\n");
const char* hybridDistanceParams =
"colfilter: \n"
" weight: 0.4 \n"
" distance: cosineangle \n"
" params: \n"
" descriptorNames: colfilter \n"
" defaultValue: 0.0 \n"
" \n"
"context: \n"
" weight: 0.4 \n"
" distance: cosineangle \n"
" params: \n"
" descriptorNames: context \n"
" defaultValue: 0.0 \n"
" \n"
"content: \n"
" weight: 0.2 \n"
" distance: euclidean \n"
" params: \n"
" descriptorNames: pca \n"
" \n";
// pick a point for doing all the queries
const Point& query = *dataset.at(1234);
const float* fquery = query.frealData().constBegin();
G_DEBUG(GTools, "query point: " << query.name() << '\n');
// ---------------------------------------------------------------------------
G_INFO('\n' << G_TITLE("USING HYBRID DISTANCE") << '\n');
ParameterMap params = toParameterMap(yaml::load(hybridDistanceParams));
DistanceFunction* dist = MetricFactory::create("LinearCombination", dataset.layout(), params);
normalSearch(dataset, dist, query, N);
// ---------------------------------------------------------------------------
frozenSearch(dataset, frozenDataSet, 175, fquery, N, BmatDistance(0.4, 0.4, 0.2));
// ---------------------------------------------------------------------------
deepFreeze(frozenDS, FrozenBmatDistance(frozenDS),
Eigen::RowVectorXf::Map(fquery, 175), N);
// ---------------------------------------------------------------------------
G_INFO("(Folding data so that it can fit into L2 cache, but doing the\n"
" exact same amount of computation as the previous line)");
G_DEBUG(GTools, "[[[ Note that the results here will be incorrect ]]]");
deepFoldedFreeze(frozenDS, FrozenBmatDistance(frozenDS),
Eigen::RowVectorXf::Map(fquery, 175), N);
// ---------------------------------------------------------------------------
multiQuery<FrozenBmatDistance, 0, 175>(frozenDS, 2);
multiQuery<FrozenBmatDistance, 0, 175>(frozenDS, 5);
multiQuery<FrozenBmatDistance, 0, 175>(frozenDS, 10);
multiQuery<FrozenBmatDistance, 0, 175>(frozenDS, 20);
// ---------------------------------------------------------------------------
G_INFO('\n' << G_TITLE("USING ONLY EUCLIDEAN DISTANCE") << '\n');
N = 100;
DistanceFunction* content = MetricFactory::create("Euclidean", dataset.layout(),
params["content"].toParameterMap()["params"].toParameterMap());
normalSearch(dataset, content, query, N);
// ---------------------------------------------------------------------------
frozenSearch(dataset, frozenDataSet+150, 175, fquery+150, N, EuclideanDistance(25));
// ---------------------------------------------------------------------------
G_INFO("(Using compacted datasets without context/colfilt data)");
frozenSearch(dataset, frozenDataSet2, 25, fquery+150, N, EuclideanDistance(25));
// ---------------------------------------------------------------------------
deepFreeze(frozeuc, FrozenEuclideanDistance(frozeuc, 0, 25),
Eigen::RowVectorXf::Map(fquery+150, 25), N);
// ---------------------------------------------------------------------------
G_INFO("(Folding data so that it can fit into L2 cache, but doing the\n"
" exact same amount of computation as the previous line)");
G_DEBUG(GTools, "[[[ Note that the results here will be incorrect ]]]");
deepFoldedFreeze(frozeuc, FrozenEuclideanDistance(frozeuc, 0, 25),
Eigen::RowVectorXf::Map(fquery+150, 25), N);
// ---------------------------------------------------------------------------
multiQuery<FrozenEuclideanDistance, 0, 25>(frozeuc, 2);
multiQuery<FrozenEuclideanDistance, 0, 25>(frozeuc, 5);
multiQuery<FrozenEuclideanDistance, 0, 25>(frozeuc, 10);
// ---------------------------------------------------------------------------
delete dist;
delete content;
delete[] frozenDataSet;
delete[] frozenDataSet2;
gaia2::shutdown();
return 0;
}