static int
MimeInlineTextHTMLSanitized_parse_line (const char *line, int32_t length,
MimeObject *obj)
{
#ifdef DEBUG_BenB
printf("p");
#endif
MimeInlineTextHTMLSanitized *textHTMLSan =
(MimeInlineTextHTMLSanitized *) obj;
#ifdef DEBUG_BenB
printf("%d", textHTMLSan->complete_buffer);
#endif
if (!textHTMLSan || !(textHTMLSan->complete_buffer))
{
#ifdef DEBUG
printf("Can't output: %s\n", line);
#endif
return -1;
}
nsCString linestr(line, length);
NS_ConvertUTF8toUTF16 line_ucs2(linestr.get());
if (length && line_ucs2.IsEmpty())
CopyASCIItoUTF16(linestr, line_ucs2);
(textHTMLSan->complete_buffer)->Append(line_ucs2);
#ifdef DEBUG_BenB
printf("l ");
#endif
return 0;
}
static int
MimeInlineTextHTMLAsPlaintext_parse_line (const char *line, PRInt32 length,
MimeObject *obj)
{
MimeInlineTextHTMLAsPlaintext *textHTMLPlain =
(MimeInlineTextHTMLAsPlaintext *) obj;
if (!textHTMLPlain || !(textHTMLPlain->complete_buffer))
{
#if DEBUG
printf("Can't output: %s\n", line);
#endif
return -1;
}
/*
To convert HTML->TXT syncronously, I need the full source at once,
not line by line (how do you convert "<li>foo\n" to plaintext?).
parse_decoded_buffer claims to give me that, but in fact also gives
me single lines.
It might be theoretically possible to drive this asyncronously, but
I don't know, which odd circumstances might arise and how libmime
will behave then. It's not worth the trouble for me to figure this all out.
*/
nsCString linestr(line, length);
NS_ConvertUTF8toUTF16 line_ucs2(linestr.get());
if (length && line_ucs2.IsEmpty())
CopyASCIItoUTF16 (linestr, line_ucs2);
(textHTMLPlain->complete_buffer)->Append(line_ucs2);
return 0;
}
std::string getUseful(const char* line) {
std::string linestr(line);
auto pos = linestr.find_first_of("//");
if (std::string::npos == pos) return linestr;
for (; pos != 0; --pos) {
if (!isblank(linestr[pos])) break;
}
for (auto i = 0; i < pos; ++pos) {
if (!isblank(linestr[i])) return linestr.substr(i, pos);
}
return "";
}
/* Function used to get the maximum attribute in both the test file & train file*/
void Adaboost::getMaxAttribute(string file_name){
/* Read the file */
ifstream filein(file_name.c_str());
set<int> attributes;
if(filein.is_open()) {
string line;
while (!filein.eof()){
getline(filein, line, '\n');
if(line == "") break;
istringstream linestr(line);
string temp;
while(linestr >> temp){
/* Get the maximum attribute */
attributes.insert(atoi(temp.substr(0, temp.find(":")).c_str()));
}
}
}
//解析行 生成node pre
//TODO
struct Pre_Node * paser_lineToPreNode(char * line,bool flag)
{
//const char * tmp = " \t";
// printf("%s\n",line);
string linestr(line);
int pos=linestr.find('{');
if(pos<0)
{
cout<<"There is illegal line!"<<endl;
exit(0);
}
struct Pre_Node *pre_node = new Pre_Node;
pre_node->main_key = linestr.substr(0,pos-1);
pre_node->query_data = linestr.substr(pos);
// cout<<pre_node->query_data;
pre_node->old_file = flag;
return pre_node;
}
int main(int argc, char* argv[])
{
if (argc != 6){
cout<< "Usuage: 5 arguments, the 1. executable followded by 2. training set, 3. training label, \
4. test set, 5. tes label, 6. Number of Classe " << endl;
exit(1);
}
int nC = atoi(argv[5]);
cout<<"Number of Classe: " <<nC<<endl;
NBC nbc;
nbc.set_numClass(nC);
//Start get Input data
ifstream traindata(argv[1]);
ifstream trainlable(argv[2]);
ifstream testdata(argv[3]);
ifstream testlable(argv[4]);
if(trainlable.is_open() && traindata.is_open()){
string line, lable;
int l =-1;
while (!trainlable.eof()){
getline(traindata, line, '\n');
getline(trainlable, lable, '\n');
istringstream linestr(line);
vector<int> sample;
int v = 0;
if (lable.size() ==0 )
break;
l = atoi(lable.c_str());
//cout << "l: " <<l <<endl;
nbc.ltrain.push_back(l);
while(linestr >> v){
sample.push_back(v);
}
nbc.trainset.push_back(sample);
}
}
void init(
std::string method,
std::string addreq,
std::string host,
std::string path, unsigned int port = 80, bool ssl = false
)
{
bool headercomplete = false;
std::set<InitParameters> seen;
while ( ! headercomplete )
{
InitParameters const initparams(method,addreq,host,path,port,ssl);
if ( seen.find(initparams) != seen.end() )
{
libmaus2::exception::LibMausException lme;
lme.getStream() << "HttpHeader: redirect loop detected, method=" << method << " host=" << host << " path=" << path << " port=" << port << " ssl=" << ssl << std::endl;
lme.finish();
throw lme;
}
seen.insert(initparams);
fields.clear();
CS.reset();
OS.reset();
SIS.reset();
GTLSIOS.reset();
SIOS = 0;
bool const hasproxy =
(ssl && hasHttpsProxy())
||
((!ssl) && hasHttpProxy());
if ( hasproxy )
{
HttpAbsoluteUrl proxyurl =
ssl ? HttpAbsoluteUrl(getenv("https_proxy")) : HttpAbsoluteUrl(getenv("http_proxy"));
// std::cerr << "[D] using proxy " << proxyurl << std::endl;
if ( proxyurl.ssl )
{
#if defined(LIBMAUS2_HAVE_GNUTLS)
libmaus2::network::GnuTLSSocket::unique_ptr_type tGTLSIOS(new libmaus2::network::GnuTLSSocket(proxyurl.host,proxyurl.port,"/etc/ssl/certs/ca-certificates.crt","/etc/ssl/certs",true));
GTLSIOS = UNIQUE_PTR_MOVE(tGTLSIOS);
SIOS = GTLSIOS.get();
#else
libmaus2::network::OpenSSLSocket::unique_ptr_type tOS(new libmaus2::network::OpenSSLSocket(proxyurl.host,proxyurl.port,0,"/etc/ssl/certs",true));
OS = UNIQUE_PTR_MOVE(tOS);
SIOS = OS.get();
#endif
libmaus2::network::SocketInputStream::unique_ptr_type tSIS(new libmaus2::network::SocketInputStream(*SIOS,64*1024));
SIS = UNIQUE_PTR_MOVE(tSIS);
}
else
{
libmaus2::network::ClientSocket::unique_ptr_type tCS(new libmaus2::network::ClientSocket(proxyurl.port,proxyurl.host.c_str()));
CS = UNIQUE_PTR_MOVE(tCS);
SIOS = CS.get();
libmaus2::network::SocketInputStream::unique_ptr_type tSIS(new libmaus2::network::SocketInputStream(*CS,64*1024));
SIS = UNIQUE_PTR_MOVE(tSIS);
}
}
else
{
if ( ssl )
{
#if defined(LIBMAUS2_HAVE_GNUTLS)
libmaus2::network::GnuTLSSocket::unique_ptr_type tGTLSIOS(new libmaus2::network::GnuTLSSocket(host,port,"/etc/ssl/certs/ca-certificates.crt","/etc/ssl/certs",true));
GTLSIOS = UNIQUE_PTR_MOVE(tGTLSIOS);
SIOS = GTLSIOS.get();
#else
libmaus2::network::OpenSSLSocket::unique_ptr_type tOS(new libmaus2::network::OpenSSLSocket(host,port,0,"/etc/ssl/certs",true));
OS = UNIQUE_PTR_MOVE(tOS);
SIOS = OS.get();
#endif
libmaus2::network::SocketInputStream::unique_ptr_type tSIS(new libmaus2::network::SocketInputStream(*SIOS,64*1024));
SIS = UNIQUE_PTR_MOVE(tSIS);
}
else
{
libmaus2::network::ClientSocket::unique_ptr_type tCS(new libmaus2::network::ClientSocket(port,host.c_str()));
CS = UNIQUE_PTR_MOVE(tCS);
SIOS = CS.get();
libmaus2::network::SocketInputStream::unique_ptr_type tSIS(new libmaus2::network::SocketInputStream(*CS,64*1024));
SIS = UNIQUE_PTR_MOVE(tSIS);
}
}
std::ostringstream reqastr;
if ( hasproxy )
{
reqastr << method << " "
<< (ssl ? "https" : "http")
<< "://"
<< host
<< ":"
<< port
<< path
<< " HTTP/1.1\r\n";
reqastr << "Connection: close\r\n";
}
else
{
reqastr << method << " " << path << " HTTP/1.1\r\n";
reqastr << "Host: " << host << "\r\n";
}
reqastr << addreq;
reqastr << "\r\n";
// send request
std::string const reqa = reqastr.str();
SIOS->write(reqa.c_str(),reqa.size());
libmaus2::autoarray::AutoArray<char> c(128,false);
char last4[4] = {0,0,0,0};
bool done = false;
std::ostringstream headstr;
while ( !done )
{
SIS->readsome(c.begin(),c.size());
ssize_t r = SIS->gcount();
// buffer is empty, try to fill it
if ( ! r )
{
SIS->read(c.begin(),1);
r = SIS->gcount();
if ( ! r )
{
libmaus2::exception::LibMausException lme;
lme.getStream() << "HttpHeader: unexpected EOF/error while reading header" << std::endl;
lme.finish();
throw lme;
}
else
{
SIS->unget();
continue;
}
}
for ( ssize_t i = 0; (!done) && i < r; ++i )
{
headstr.put(c[i]);
last4[0] = last4[1];
last4[1] = last4[2];
last4[2] = last4[3];
last4[3] = c[i];
if (
last4[0] == '\r' &&
last4[1] == '\n' &&
last4[2] == '\r' &&
last4[3] == '\n'
)
{
for ( ssize_t j = i+1; j < r; ++j )
SIS->unget();
// SIS->clear();
done = true;
}
}
}
std::istringstream linestr(headstr.str());
std::vector<std::string> lines;
while ( linestr )
{
std::string line;
std::getline(linestr,line);
while ( line.size() && isspace(line[line.size()-1]) )
{
line = line.substr(0,line.size()-1);
}
if ( line.size() )
lines.push_back(line);
}
if ( ! lines.size() )
{
libmaus2::exception::LibMausException lme;
lme.getStream() << "HttpHeader: unable to get header" << std::endl;
lme.finish();
throw lme;
}
statusline = lines[0];
std::vector<std::string> statustokens;
for ( uint64_t i = 0; i < statusline.size(); )
{
while ( i < statusline.size() && isspace(statusline[i]) )
++i;
uint64_t j = i;
while ( j < statusline.size() && (!isspace(statusline[j])) )
++j;
std::string const token = statusline.substr(i,j-i);
i = j;
statustokens.push_back(token);
}
if ( statustokens.size() < 2 )
{
libmaus2::exception::LibMausException lme;
lme.getStream() << "HttpHeader: status line " << statusline << " invalid" << std::endl;
lme.finish();
throw lme;
}
std::string const replyformat = statustokens[0];
std::istringstream statuscodeistr(statustokens[1]);
uint64_t statuscode;
statuscodeistr >> statuscode;
if ( ! statuscodeistr )
{
libmaus2::exception::LibMausException lme;
lme.getStream() << "HttpHeader: invalid status code " << statustokens[1] << std::endl;
lme.finish();
throw lme;
}
if (
replyformat != "HTTP/1.0"
&&
replyformat != "HTTP/1.1"
)
{
libmaus2::exception::LibMausException lme;
lme.getStream() << "HttpHeader: unknown reply format " << replyformat << std::endl;
lme.finish();
throw lme;
}
// std::cerr << "status code " << statuscode << std::endl;
for ( uint64_t i = 1; i < lines.size(); ++i )
{
std::string const line = lines[i];
uint64_t col = 0;
while ( col < line.size() && line[col] != ':' )
++col;
if ( ! col || col == line.size() )
{
libmaus2::exception::LibMausException lme;
lme.getStream() << "HttpHeader: invalid key value pair " << line << std::endl;
lme.finish();
throw lme;
}
std::string const key = tolower(despace(line.substr(0,col)));
std::string const val = despace(line.substr(col+1));
fields[key] = val;
}
switch ( statuscode )
{
case 301:
case 302:
case 307:
case 308:
{
std::map<std::string,std::string>::const_iterator it = fields.end();
if ( it == fields.end() )
it = fields.find("Location");
if ( it == fields.end() )
it = fields.find("location");
if ( it == fields.end() )
{
libmaus2::exception::LibMausException lme;
lme.getStream() << "HttpHeader: redirect status code " << statuscode << " but no location given" << std::endl;
lme.finish();
throw lme;
}
std::string location = it->second;
// std::cerr << "redirecting " << statuscode << " to " << location << std::endl;
if (
::libmaus2::network::HttpAbsoluteUrl::isHttpAbsoluteUrl(location)
||
::libmaus2::network::HttpAbsoluteUrl::isHttpsAbsoluteUrl(location)
)
{
::libmaus2::network::HttpAbsoluteUrl url(location);
host = url.host;
port = url.port;
path = url.path;
ssl = url.ssl;
}
else if ( ::libmaus2::network::HttpAbsoluteUrl::isAbsoluteUrl(location) )
{
libmaus2::exception::LibMausException lme;
lme.getStream() << "HttpHeader: unsupported protocol in location " << location << std::endl;
lme.finish();
throw lme;
}
else
{
path = location;
}
break;
}
default:
{
headercomplete = true;
break;
}
}
}
#if 0
std::cerr << statusline << std::endl;
for ( std::map<std::string,std::string>::const_iterator ita = fields.begin(); ita != fields.end(); ++ita )
{
std::cerr << ita->first << ": " << ita->second << std::endl;
}
#endif
url.host = host;
url.port = port;
url.path = path;
url.ssl = ssl;
}
