template<class ST> bool CDenseFeatures<ST>::apply_preprocessor(bool force_preprocessing)
{
if (m_subset_stack->has_subsets())
SG_ERROR("A subset is set, cannot call apply_preproc\n")
SG_DEBUG("force: %d\n", force_preprocessing)
if (feature_matrix.matrix && get_num_preprocessors())
{
for (int32_t i = 0; i < get_num_preprocessors(); i++)
{
if ((!is_preprocessed(i) || force_preprocessing))
{
set_preprocessed(i);
CDensePreprocessor<ST>* p =
(CDensePreprocessor<ST>*) get_preprocessor(i);
SG_INFO("preprocessing using preproc %s\n", p->get_name())
if (p->apply_to_feature_matrix(this).matrix == NULL)
{
SG_UNREF(p);
return false;
}
SG_UNREF(p);
}
}
return true;
}
template<class ST> ST* CDenseFeatures<ST>::get_feature_vector(int32_t num, int32_t& len, bool& dofree)
{
/* index conversion for subset, only for array access */
int32_t real_num=m_subset_stack->subset_idx_conversion(num);
len = num_features;
if (feature_matrix.matrix)
{
dofree = false;
return &feature_matrix.matrix[real_num * int64_t(num_features)];
}
ST* feat = NULL;
dofree = false;
if (feature_cache)
{
feat = feature_cache->lock_entry(real_num);
if (feat)
return feat;
else
feat = feature_cache->set_entry(real_num);
}
if (!feat)
dofree = true;
feat = compute_feature_vector(num, len, feat);
if (get_num_preprocessors())
{
int32_t tmp_len = len;
ST* tmp_feat_before = feat;
ST* tmp_feat_after = NULL;
for (int32_t i = 0; i < get_num_preprocessors(); i++)
{
CDensePreprocessor<ST>* p =
(CDensePreprocessor<ST>*) get_preprocessor(i);
// temporary hack
SGVector<ST> applied = p->apply_to_feature_vector(
SGVector<ST>(tmp_feat_before, tmp_len));
tmp_feat_after = applied.vector;
SG_UNREF(p);
if (i != 0) // delete feature vector, except for the the first one, i.e., feat
SG_FREE(tmp_feat_before);
tmp_feat_before = tmp_feat_after;
}
memcpy(feat, tmp_feat_after, sizeof(ST) * tmp_len);
SG_FREE(tmp_feat_after);
len = tmp_len;
}
return feat;
}
free_sparse_feature_matrix();
sparse_feature_matrix.from_dense(full);
}
template<class ST> bool CSparseFeatures<ST>::apply_preprocessor(bool force_preprocessing)
{
SG_INFO("force: %d\n", force_preprocessing)
if (sparse_feature_matrix.sparse_matrix && get_num_preprocessors())
{
for (int32_t i=0; i<get_num_preprocessors(); i++)
{
if (!is_preprocessed(i) || force_preprocessing)
{
set_preprocessed(i);
CSparsePreprocessor<ST>* p = (CSparsePreprocessor<ST>*) get_preprocessor(i);
SG_INFO("preprocessing using preproc %s\n", p->get_name())
if (p->apply_to_sparse_feature_matrix(this) == NULL)
{
SG_UNREF(p);
return false;
}
SG_UNREF(p);
}
}
return true;
}
else
{
std::vector<cmonster::core::Token>
FunctionMacro::operator()(
clang::SourceLocation const& expansion_location,
std::vector<cmonster::core::Token> const& arguments) const
{
// Create the arguments tuple.
ScopedPyObject args_tuple = PyTuple_New(arguments.size());
if (!args_tuple)
throw std::runtime_error("Failed to create argument tuple");
for (Py_ssize_t i = 0; i < static_cast<Py_ssize_t>(arguments.size()); ++i)
{
Token *token = create_token(m_preprocessor, arguments[i]);
PyTuple_SetItem(args_tuple, i, reinterpret_cast<PyObject*>(token));
}
// Set the "preprocessor" and "location" global variables.
//
// XXX How do we create a closure via the C API? It would be better if we
// could bind a function to the preprocessor it was created with, when we
// define the function.
PyObject *globals = PyEval_GetGlobals();
if (globals)
{
PyObject *key = PyUnicode_FromString("preprocessor");
if (!key)
throw python_exception();
Py_INCREF((PyObject*)m_preprocessor);
PyDict_SetItem(globals, key, (PyObject*)m_preprocessor);
cmonster::core::Preprocessor &pp = get_preprocessor(m_preprocessor);
SourceLocation *location = create_source_location(
expansion_location, pp.getClangPreprocessor().getSourceManager());
if (!location)
throw python_exception();
key = PyUnicode_FromString("location");
if (!key)
throw python_exception();
PyDict_SetItem(globals, key, (PyObject*)location);
}
// Call the function.
ScopedPyObject py_result = PyObject_Call(m_callable, args_tuple, NULL);
if (!py_result)
throw python_exception();
// Transform the result.
std::vector<cmonster::core::Token> result;
if (py_result == Py_None)
return result;
// Is it a string? If so, tokenize it.
if (PyUnicode_Check(py_result))
{
ScopedPyObject utf8(PyUnicode_AsUTF8String(py_result));
if (utf8)
{
char *u8_chars;
Py_ssize_t u8_size;
if (PyBytes_AsStringAndSize(utf8, &u8_chars, &u8_size) == -1)
{
throw python_exception();
}
else
{
cmonster::core::Preprocessor &pp =
get_preprocessor(m_preprocessor);
result = pp.tokenize(u8_chars, u8_size);
return result;
}
}
else
{
throw python_exception();
}
}
// If it's not a string, it should be a sequence of Token objects.
if (!PySequence_Check(py_result))
{
throw python_exception(PyExc_TypeError,
"macro functions must return a sequence of tokens");
}
const Py_ssize_t seqlen = PySequence_Size(py_result);
if (seqlen == -1)
{
throw python_exception();
}
else
{
for (Py_ssize_t i = 0; i < seqlen; ++i)
{
ScopedPyObject token_ = PySequence_GetItem(py_result, i);
if (PyObject_TypeCheck(token_, get_token_type()))
{
Token *token = (Token*)(PyObject*)token_;
result.push_back(get_token(token));
}
else
{
// Invalid return value.
throw python_exception(PyExc_TypeError,
"macro functions must return a sequence of tokens");
}
}
}
return result;
}
