// download body for this request
int fancydm::in(DataBuffer * d, Socket * sock, Socket * peersock, class HTTPHeader * requestheader, class HTTPHeader * docheader, bool wantall, int *headersent, bool * toobig)
{
//DataBuffer *d = where to stick the data back into
//Socket *sock = where to read from
//Socket *peersock = browser to send stuff to for keeping it alive
//HTTPHeader *docheader = header used for sending first line of reply
//HTTPHeader *requestheader = header client used to request
//bool wantall = to determine if just content filter or a full scan
//int *headersent = to use to send the first line of header if needed
// or to mark the header has already been sent
//bool *toobig = flag to modify to say if it could not all be downloaded
#ifdef DGDEBUG
std::cout << "Inside fancy download manager plugin" << std::endl;
#endif
int rc;
off_t newsize;
off_t expectedsize = docheader->contentLength();
off_t bytessec = 0;
off_t bytesgot = 0;
int percentcomplete = 0;
unsigned int eta = 0;
int timeelapsed = 0;
// if using non-persistent connections, some servers will not report
// a content-length. in these situations, just download everything.
bool geteverything = false;
if ((expectedsize < 0) && !(docheader->isPersistent()))
geteverything = true;
if (expectedsize < 0)
expectedsize = 0;
bool initialsent = false;
String message, jsmessage;
char *block = NULL; // buffer for storing a grabbed block from the input stream
char *temp = NULL;
bool swappedtodisk = false;
struct timeval starttime;
struct timeval themdays;
struct timeval nowadays;
gettimeofday(&themdays, NULL);
gettimeofday(&starttime, NULL);
toobig_unscanned = false;
toobig_notdownloaded = false;
bool secondstage = false;
// buffer size for streaming downloads
off_t blocksize = 32768;
// set to a sensible minimum
if (!wantall && (blocksize > o.max_content_filter_size))
blocksize = o.max_content_filter_size;
else if (wantall && (blocksize > o.max_content_ramcache_scan_size))
blocksize = o.max_content_ramcache_scan_size;
#ifdef DGDEBUG
std::cout << "blocksize: " << blocksize << std::endl;
#endif
// determine downloaded filename
String filename(requestheader->disposition());
if (filename.length() == 0) {
filename = requestheader->getUrl();
filename = requestheader->decode(filename);
if (filename.contains("?"))
filename = filename.before("?");
while (filename.contains("/"))
filename = filename.after("/");
}
while ((bytesgot < expectedsize) || geteverything) {
// send text header to show status
if (o.trickle_delay > 0) {
gettimeofday(&nowadays, NULL);
timeelapsed = nowadays.tv_sec - starttime.tv_sec;
if ((!initialsent && timeelapsed > o.initial_trickle_delay) || (initialsent && nowadays.tv_sec - themdays.tv_sec > o.trickle_delay)) {
initialsent = true;
bytessec = bytesgot / timeelapsed;
themdays.tv_sec = nowadays.tv_sec;
if ((*headersent) < 1) {
#ifdef DGDEBUG
std::cout << "sending header for text status" << std::endl;
#endif
message = "HTTP/1.0 200 OK\nContent-Type: text/html\n\n";
// Output initial template
std::deque<String>::iterator i = progresspage.html.begin();
std::deque<String>::iterator penultimate = progresspage.html.end()-1;
bool newline;
while (i != progresspage.html.end()) {
newline = false;
message = *i;
if (message == "-FILENAME-") {
message = filename;
}
else if (message == "-FILESIZE-") {
message = String(expectedsize);
}
else if (message == "-SERVERIP-") {
message = peersock->getLocalIP();
}
else if ((i == penultimate) || ((*(i+1))[0] != '-')) {
newline = true;
}
peersock->writeString(message.toCharArray());
// preserve line breaks from the original template file
if (newline)
peersock->writeString("\n");
i++;
}
// send please wait message for non-JS-enabled browsers
// 1200 "Please wait - downloading to be scanned..."
message = "<noscript><p>";
message += o.language_list.getTranslation(1200);
message += "</p></noscript>\n";
peersock->writeString(message.toCharArray());
(*headersent) = 2;
}
#ifdef DGDEBUG
std::cout << "trickle delay - sending progress..." << std::endl;
#endif
message = "Downloading status: ";
// Output a call to template's JavaScript progressupdate function
jsmessage = "<script language='javascript'>\n<!--\nprogressupdate(" + String(bytesgot) + "," + String(bytessec) + ");\n//-->\n</script>";
peersock->writeString(jsmessage.toCharArray());
// send text only version for non-JS-enabled browsers.
// checkme: translation?
if (geteverything) {
message = "<noscript><p>Time remaining: unknown; "
+ bytestring(bytessec) + "/s; total downloaded: " + bytestring(bytesgot) + "</p></noscript>\n";
} else {
percentcomplete = bytesgot/(expectedsize/100);
eta = (expectedsize-bytesgot)/bytessec;
message = "<noscript><p>" + String(percentcomplete) + "%, time remaining: " + timestring(eta) + "; "
+ bytestring(bytessec) + "/s; total downloaded: " + bytestring(bytesgot) + "</p></noscript>\n";
}
peersock->writeString(message.toCharArray());
peersock->writeString("<!-- force flush -->\r\n");
}
}
if (wantall) {
if (!swappedtodisk) {
// if not swapped to disk and file is too large for RAM, then swap to disk
if (bytesgot > o.max_content_ramcache_scan_size) {
#ifdef DGDEBUG
std::cout << "swapping to disk" << std::endl;
#endif
d->tempfilefd = d->getTempFileFD();
if (d->tempfilefd < 0) {
#ifdef DGDEBUG
std::cerr << "error buffering to disk so skipping disk buffering" << std::endl;
#endif
syslog(LOG_ERR, "%s", "error buffering to disk so skipping disk buffering");
(*toobig) = true;
break;
}
writeEINTR(d->tempfilefd, d->data, d->buffer_length);
swappedtodisk = true;
d->tempfilesize = d->buffer_length;
}
} else if (bytesgot > o.max_content_filecache_scan_size) {
(*toobig) = true;
toobig_unscanned = true;
if (geteverything && (upperlimit > 0)) {
// multi-stage download enabled, and we don't know content length
if ((!secondstage) && initialsent) {
secondstage = true;
// send download size warning message
jsmessage = "<script language='javascript'>\n<!--\ndownloadwarning(" + String(upperlimit) + ");\n//-->\n</script>";
peersock->writeString(jsmessage.toCharArray());
// text-only version
message = "<noscript><p>";
// 1201 Warning: file too large to scan. If you suspect that this file is larger than
// 1202 , then refresh to download directly.
message += o.language_list.getTranslation(1201);
message += bytestring(upperlimit);
message += o.language_list.getTranslation(1202);
message += "</p></noscript>\n";
peersock->writeString(message.toCharArray());
peersock->writeString("<!-- force flush -->\r\n");
// add URL to clean cache (for all groups)
String url(requestheader->getUrl());
addToClean(url, o.filter_groups + 1);
#ifdef DGDEBUG
std::cout << "fancydm: file too big to be scanned, entering second stage of download" << std::endl;
#endif
}
// too large to even download, let alone scan
if (bytesgot > upperlimit) {
#ifdef DGDEBUG
std::cout << "fancydm: file too big to be downloaded, halting second stage of download" << std::endl;
#endif
toobig_unscanned = false;
toobig_notdownloaded = true;
break;
}
} else {
// multi-stage download disabled, or we know content length
// if swapped to disk and file too large for that too, then give up
#ifdef DGDEBUG
std::cout << "fancydm: file too big to be scanned, halting download" << std::endl;
#endif
toobig_unscanned = false;
toobig_notdownloaded = true;
break;
}
}
} else {
if (bytesgot > o.max_content_filter_size) {
// if we aren't downloading for virus scanning, and file too large for filtering, give up
#ifdef DGDEBUG
std::cout << "fancydm: file too big to be filtered, halting download" << std::endl;
#endif
(*toobig) = true;
break;
}
}
if (!swappedtodisk) {
if (d->buffer_length >= blocksize) {
newsize = d->buffer_length;
} else {
newsize = blocksize;
}
#ifdef DGDEBUG
std::cout << "newsize: " << newsize << std::endl;
#endif
// if not getting everything until connection close, grab only what is left
if (!geteverything && (newsize > (expectedsize - bytesgot)))
newsize = expectedsize - bytesgot;
delete[] block;
block = new char[newsize];
try {
sock->checkForInput(d->timeout);
} catch(std::exception & e) {
break;
}
// improved more efficient socket read which uses the buffer better
rc = d->bufferReadFromSocket(sock, block, newsize, d->timeout, o.trickle_delay);
// grab a block of input, doubled each time
if (rc <= 0) {
break; // an error occured so end the while()
// or none received so pipe is closed
}
else {
/*if (d->data != temp)
delete[] temp;*/
temp = new char[d->buffer_length + rc + 1]; // replacement store
temp[d->buffer_length + rc] = '\0';
memcpy(temp, d->data, d->buffer_length); // copy the current data
memcpy(temp + d->buffer_length, block, rc); // copy the new data
delete[]d->data; // delete the current data block
d->data = temp;
temp = NULL;
d->buffer_length += rc; // update data size counter
}
} else {
try {
sock->checkForInput(d->timeout);
}
catch(std::exception & e) {
break;
}
rc = d->bufferReadFromSocket(sock, d->data,
// if not getting everything until connection close, grab only what is left
(!geteverything && ((expectedsize - bytesgot) < d->buffer_length) ? (expectedsize - bytesgot) : d->buffer_length), d->timeout);
if (rc <= 0) {
break;
}
else {
lseek(d->tempfilefd, 0, SEEK_END); // not really needed
writeEINTR(d->tempfilefd, d->data, rc);
d->tempfilesize += rc;
#ifdef DGDEBUG
std::cout << "written to disk: " << rc << " total: " << d->tempfilesize << std::endl;
#endif
}
}
if (d->tempfilesize > 0) {
bytesgot = d->tempfilesize;
} else {
bytesgot = d->buffer_length;
}
}
if (initialsent) {
if (!swappedtodisk) { // if we sent textual content then we can't
// stream the file to the user so we must save to disk for them
// to download by clicking on the magic link
// You can get to this point by having a large ram cache, or
// slow internet connection with small initial trickle delay.
// This should be rare.
#ifdef DGDEBUG
std::cout << "swapping to disk" << std::endl;
#endif
d->tempfilefd = d->getTempFileFD();
if (d->tempfilefd < 0) {
#ifdef DGDEBUG
std::cerr << "error buffering complete to disk so skipping disk buffering" << std::endl;
#endif
syslog(LOG_ERR, "error buffering complete to disk so skipping disk buffering");
} else {
writeEINTR(d->tempfilefd, d->data, d->buffer_length);
swappedtodisk = true;
d->tempfilesize = d->buffer_length;
}
}
// Output a call to template's JavaScript nowscanning function
peersock->writeString("<script language='javascript'>\n<!--\nnowscanning();\n//-->\n</script>\n");
// send text-only version
// 1210 "Download Complete. Starting scan..."
if (!(toobig_unscanned || toobig_notdownloaded)) {
message = "<noscript><p>";
message += o.language_list.getTranslation(1210);
message += "</p></noscript>\n";
peersock->writeString(message.toCharArray());
}
// only keep full downloads
if (!toobig_notdownloaded)
(*d).preservetemp = true;
(*d).dontsendbody = true;
}
if (!(*toobig) && !swappedtodisk) { // won't deflate stuff swapped to disk
if (d->decompress.contains("deflate")) {
#ifdef DGDEBUG
std::cout << "zlib format" << std::endl;
#endif
d->zlibinflate(false); // incoming stream was zlib compressed
}
else if (d->decompress.contains("gzip")) {
#ifdef DGDEBUG
std::cout << "gzip format" << std::endl;
#endif
d->zlibinflate(true); // incoming stream was gzip compressed
}
}
d->bytesalreadysent = 0;
/*if (d->data != temp)
delete[] temp;*/
delete[] block;
return 0;
}
// download body for this request
int dminstance::in(DataBuffer *d, Socket *sock, Socket *peersock, class HTTPHeader *requestheader,
class HTTPHeader *docheader, bool wantall, int *headersent, bool *toobig)
{
//DataBuffer *d = where to stick the data back into
//Socket *sock = where to read from
//Socket *peersock = browser to send stuff to for keeping it alive
//HTTPHeader *requestheader = header client used to request
//HTTPHeader *docheader = header used for sending first line of reply
//bool wantall = to determine if just content filter or a full scan
//int *headersent = to use to send the first line of header if needed
// or to mark the header has already been sent
//bool *toobig = flag to modify to say if it could not all be downloaded
#ifdef DGDEBUG
std::cout << "Inside default download manager plugin" << std::endl;
#endif
// To access settings for the plugin use the following example:
// std::cout << "cvtest:" << cv["dummy"] << std::endl;
int rc;
off_t newsize;
off_t bytesremaining = docheader->contentLength();
// if using non-persistent connections, some servers will not report
// a content-length. in these situations, just download everything.
bool geteverything = false;
if ((bytesremaining < 0) && !(docheader->isPersistent()))
geteverything = true;
char *block = NULL; // buffer for storing a grabbed block from the
// imput stream
char *temp = NULL;
bool swappedtodisk = false;
bool doneinitialdelay = false;
struct timeval themdays;
struct timeval nowadays;
gettimeofday(&themdays, NULL);
// buffer size for streaming downloads
off_t blocksize = 32768;
// set to a sensible minimum
if (!wantall && (blocksize > o.max_content_filter_size))
blocksize = o.max_content_filter_size;
else if (wantall && (blocksize > o.max_content_ramcache_scan_size))
blocksize = o.max_content_ramcache_scan_size;
#ifdef DGDEBUG
std::cout << "blocksize: " << blocksize << std::endl;
#endif
while ((bytesremaining > 0) || geteverything) {
// send x-header keep-alive here
if (o.trickle_delay > 0) {
gettimeofday(&nowadays, NULL);
if (doneinitialdelay ? nowadays.tv_sec - themdays.tv_sec > o.trickle_delay : nowadays.tv_sec - themdays.tv_sec > o.initial_trickle_delay) {
themdays.tv_sec = nowadays.tv_sec;
doneinitialdelay = true;
if ((*headersent) < 1) {
#ifdef DGDEBUG
std::cout << "sending first line of header first" << std::endl;
#endif
docheader->out(NULL, peersock, __DGHEADER_SENDFIRSTLINE);
(*headersent) = 1;
}
#ifdef DGDEBUG
std::cout << "trickle delay - sending X-DGKeepAlive: on" << std::endl;
#endif
peersock->writeString("X-DGKeepAlive: on\r\n");
}
}
if (wantall) {
if (!swappedtodisk) {
// if not swapped to disk and file is too large for RAM, then swap to disk
if (d->buffer_length > o.max_content_ramcache_scan_size) {
#ifdef DGDEBUG
std::cout << "swapping to disk" << std::endl;
#endif
d->tempfilefd = d->getTempFileFD();
if (d->tempfilefd < 0) {
#ifdef DGDEBUG
std::cerr << "error buffering to disk so skipping disk buffering" << std::endl;
#endif
syslog(LOG_ERR, "%s", "error buffering to disk so skipping disk buffering");
(*toobig) = true;
break;
}
writeEINTR(d->tempfilefd, d->data, d->buffer_length);
swappedtodisk = true;
d->tempfilesize = d->buffer_length;
}
} else if (d->tempfilesize > o.max_content_filecache_scan_size) {
// if swapped to disk and file too large for that too, then give up
#ifdef DGDEBUG
std::cout << "defaultdm: file too big to be scanned, halting download" << std::endl;
#endif
(*toobig) = true;
break;
}
} else {
if (d->buffer_length > o.max_content_filter_size) {
// if we aren't downloading for virus scanning, and file too large for filtering, give up
#ifdef DGDEBUG
std::cout << "defaultdm: file too big to be filtered, halting download" << std::endl;
#endif
(*toobig) = true;
break;
}
}
if (!swappedtodisk) {
if (d->buffer_length >= blocksize) {
newsize = d->buffer_length;
} else {
newsize = blocksize;
}
#ifdef DGDEBUG
std::cout << "newsize: " << newsize << std::endl;
#endif
// if not getting everything until connection close, grab only what is left
if (!geteverything && (newsize > bytesremaining))
newsize = bytesremaining;
delete[] block;
block = new char[newsize];
try {
sock->checkForInput(d->timeout);
} catch (std::exception &e) {
break;
}
// improved more efficient socket read which uses the buffer better
rc = d->bufferReadFromSocket(sock, block, newsize, d->timeout);
// grab a block of input, doubled each time
if (rc <= 0) {
break; // an error occured so end the while()
// or none received so pipe is closed
} else {
bytesremaining -= rc;
/*if (d->data != temp)
delete[] temp;*/
temp = new char[d->buffer_length + rc + 1]; // replacement store
temp[d->buffer_length + rc] = '\0';
memcpy(temp, d->data, d->buffer_length); // copy the current data
memcpy(temp + d->buffer_length, block, rc); // copy the new data
delete[] d->data; // delete the current data block
d->data = temp;
temp = NULL;
d->buffer_length += rc; // update data size counter
}
} else {
try {
sock->checkForInput(d->timeout);
} catch (std::exception &e) {
break;
}
rc = d->bufferReadFromSocket(sock, d->data,
// if not getting everything until connection close, grab only what is left
(!geteverything && (bytesremaining < d->buffer_length) ? bytesremaining : d->buffer_length), d->timeout);
if (rc <= 0) {
break;
} else {
bytesremaining -= rc;
lseek(d->tempfilefd, 0, SEEK_END); // not really needed
writeEINTR(d->tempfilefd, d->data, rc);
d->tempfilesize += rc;
#ifdef DGDEBUG
std::cout << "written to disk:" << rc << " total:" << d->tempfilesize << std::endl;
#endif
}
}
}
if (!(*toobig) && !swappedtodisk) { // won't deflate stuff swapped to disk
if (d->decompress.contains("deflate")) {
#ifdef DGDEBUG
std::cout << "zlib format" << std::endl;
#endif
d->zlibinflate(false); // incoming stream was zlib compressed
} else if (d->decompress.contains("gzip")) {
#ifdef DGDEBUG
std::cout << "gzip format" << std::endl;
#endif
d->zlibinflate(true); // incoming stream was gzip compressed
}
}
d->bytesalreadysent = 0;
#ifdef DGDEBUG
std::cout << "Leaving default download manager plugin" << std::endl;
#endif
delete[] block;
/*if (d->data != temp)
delete[] temp;*/
return 0;
}
