OverInfo *
FindCanceledMsg(const char *group, const char *msgid, artno_t *partNo, int *pvalidGroups)
{
const char *rec;
int recLen;
OverInfo *ov = NULL;
/*
* Make sure group is valid before calling GetOverInfo() or we will
* create random over. files for illegal groups. Don't lock the record,
* meaning that we have to re-read it after calling GetOverInfo (otherwise
* someone else can update the numeric fields in the record while we are
* trying to process them).
*/
*partNo = -1;
if ((rec = KPDBReadRecord(KDBActive, group, 0, &recLen)) != NULL) {
++*pvalidGroups;
if ((ov = GetOverInfo(group)) != NULL) {
hash_t hv = hhash(msgid);
if ((rec = KPDBReadRecord(KDBActive, group, KP_LOCK, &recLen)) != NULL) {
artno_t artBeg;
artno_t artEnd;
artBeg = strtoll(KPDBGetField(rec, recLen, "NB", NULL, "-1"), NULL, 10);
artEnd = strtoll(KPDBGetField(rec, recLen, "NE", NULL, "-1"), NULL, 10);
if (artEnd - artBeg > ov->ov_MaxArts)
artBeg = artEnd - ov->ov_MaxArts;
while (artEnd >= artBeg) {
off_t ovpos = 0;
const OverArt *oa = GetOverArt(ov, artEnd, &ovpos);
if (OA_ARTNOEQ(artEnd, oa->oa_ArtNo) &&
bcmp(&oa->oa_MsgHash, &hv, sizeof(hv)) == 0
) {
*partNo = artEnd;
break;
}
--artEnd;
} /* while */
KPDBUnlock(KDBActive, rec);
}
if (*partNo == -1) {
PutOverInfo(ov);
ov = NULL;
}
} /* if */
}
return(ov);
}
int
main(int ac, char **av)
{
int r = 0;
int i;
char *arg = NULL;
char *file = NULL;
LogFo = stderr;
LoadDiabloConfig(ac, av);
for (i = 1; i < ac; ++i) {
char *ptr = av[i];
if (*ptr != '-') {
arg = ptr;
continue;
}
ptr += 2;
switch(ptr[-1]) {
case 'C':
if (*ptr == 0)
++i;
break;
case 'd':
DebugOpt = atoi(*ptr ? ptr : av[++i]);
break;
case 'F':
ForceOpt = 1;
break;
case 'f':
file = (*ptr) ? ptr : av[++i];
break;
case 'H':
ShowFileHeader = 1;
break;
case 'h':
HeadOnly = 1;
break;
case 'q':
QuietOpt = 1;
break;
case 's':
StripCR = 0;
break;
case 'V':
PrintVersion();
break;
case 'v':
VerifyOnly = 1;
LogFo = stdout;
break;
default:
fprintf(stderr, "dreadart: Illegal option: %s\n", ptr - 2);
Usage(av[0]);
}
}
if (arg == NULL && file == NULL)
Usage(av[0]);
HistoryOpen(NULL, HGF_READONLY);
LoadSpoolCtl(0, 1);
if (arg == NULL) {
char buf[8192];
char msgid[MAXMSGIDLEN];
FILE *fi = (strcmp(file, "-") == 0) ? stdin : fopen(file, "r");
if (fi) {
while (fgets(buf, sizeof(buf), fi) != NULL) {
hash_t hv;
char *m;
if (strncmp(buf, "DUMP ", 5) == 0) {
History h = { 0 };
if (sscanf(buf + 5, "%s gm=%d ex=%hd boff=%d bsize=%d",
msgid, &h.gmt, &h.exp,
&h.boffset, &h.bsize) == 5) {
char *p;
h.hv.h1 = (int32)strtoul(msgid, &p, 16);
if (*p == '.')
h.hv.h2 = (int32)strtoul(p + 1, &p, 16);
if (*p == '.')
h.iter = (int16)strtoul(p + 1, NULL, 16);
r = LookupHash(h.hv, NULL, &h);
}
} else if ((m = strchr(buf, '<')) != NULL) {
char *p;
if ((p = strchr(m, '>')) == NULL)
continue;
*++p = 0;
hv = hhash(m);
r = LookupHash(hv, m, NULL);
} else if (sscanf(buf, "%x.%x", &hv.h1, &hv.h2) == 2) {
r = LookupHash(hv, NULL, NULL);
}
}
if (fi != stdin)
fclose(fi);
} else {
fprintf(stderr, "Unable to open %s (%s)\n", file, strerror(errno));
}
} else {
hash_t hv;
char *msgid = NULL;
if (arg[0] == '<') {
msgid = arg;
hv = hhash(arg);
r = LookupHash(hv, msgid, NULL);
} else if (arg[0] == 'D' && arg[1] == '.') {
int32 dummy;
if (sscanf(arg + 2, "%x/%x.%x", &dummy, &hv.h1, &hv.h2) != 3) {
fprintf(stderr, "argument error\n");
exit(1);
}
r = LookupHash(hv, msgid, NULL);
} else if (sscanf(arg, "%x.%x", &hv.h1, &hv.h2) == 2) {
r = LookupHash(hv, msgid, NULL);
} else {
char fname[PATH_MAX];
char *p = fname;
History h = { 0 };
*p = 0;
if (*arg != '/') {
sprintf(p, "%s/", PatExpand(SpoolHomePat));
p += strlen(p);
}
if (sscanf(arg, "%[^:]:%d,%d", p, &h.boffset, &h.bsize) == 3) {
DumpArticle(fname, &h, NULL);
} else {
printf("Unknown argument: %s\n", arg);
}
}
}
exit(r);
}
void
ScanSpoolFileMapOld(const char *base, int bytes, int gmt, int iter, char *dpath, uint16 spoolobj)
{
int b = 0;
int count = 0;
/*
* scan file
*/
printf(" (old format) ");
while (b < bytes) {
int i = b;
int inHeader = 1;
int linesLeft = -1;
int numLines = 0;
msgId[0] = 0;
newsgroups[0] = 0;
/*
* scan article
*/
while (i < bytes && linesLeft && base[i] != 0) {
int l;
/*
* Scan line
*/
for (l = i; l < bytes; l++) {
if (base[l] == '\n') {
l++;
if (l == i + 1 || !inHeader)
break;
if (l < bytes && base[l] != ' ' && base[l] != '\t')
break;
l--;
}
}
if (inHeader) {
if (l - i == 1) {
inHeader = 0;
} else if (strncasecmp(base + i, "Lines:", 6) == 0) {
linesLeft = strtol(base + i + 6, NULL, 0);
} else if (strncasecmp(base + i, "Message-ID:", 11) == 0) {
diablo_strlcpynl(msgId, base + i + 11, l - i - 11, sizeof(msgId));
} else if (strncasecmp(base + i, "Newsgroups:", 11) == 0) {
diablo_strlcpynl2(newsgroups, ',', base + i + 11, l - i - 11, sizeof(newsgroups));
}
} else {
--linesLeft;
++numLines;
}
i = l;
}
if (i < bytes && base[i] == 0) {
const char *id = MsgId(msgId, NULL);
History h = { 0 };
h.hv = hhash(id);
h.iter = iter;
h.gmt = gmt;
h.exp = 100 + spoolobj;
h.boffset = b;
h.bsize = i - b;
if (numLines == 0)
h.exp |= EXPF_HEADONLY;
DoArticle(&h, id, newsgroups, " ", " ", 0, " ", " ");
count++;
++i;
} else {
if (!QuietOpt) {
printf("\tFailed %d,%d %s\n", b, i - b, MsgId(msgId, NULL));
fflush(stdout);
}
/*
write(1, base + b, i - b);
write(1, "*", 1);
printf("(%d)\n", base[i]);
*/
while (i < bytes && base[i] != 0)
++i;
if (i < bytes)
++i;
}
b = i;
}
if (!QuietOpt)
printf("%d entries", count);
}
void
ScanSpoolFileMap(const char *base, int bytes, int gmt, int iter, char *dpath, uint16 spoolobj, int fd)
{
int count = 0;
int b = 0;
int arthdrlen;
char *artbase = NULL;
char *artpos;
SpoolArtHdr ah;
char cSize[64];
int headOnly;
while (b < bytes) {
bcopy(base + b, &ah, sizeof(ah));
if ((uint8)ah.Magic1 != STORE_MAGIC1 ||
(uint8)ah.Magic2 != STORE_MAGIC2) {
printf("\tFailed at offset %d: invalid header magic (%d:%d)\n", b,
ah.Magic1, ah.Magic2);
ScanSpoolFileMapOld(base + b, bytes - b, gmt, iter, dpath, spoolobj);
return;
}
arthdrlen = ah.ArtHdrLen;
if (ah.StoreType & STORETYPE_GZIP) {
#ifdef USE_ZLIB
gzFile *gzf;
long len = ah.ArtLen;
artbase = (char *)malloc(ah.ArtLen + 2);
bzero(artbase, ah.ArtLen + 2);
lseek(fd, b + ah.HeadLen, 0);
if ((gzf = gzdopen(dup(fd), "r")) != NULL) {
if (gzread(gzf, artbase, len) != len)
arthdrlen = 0;
gzclose(gzf);
} else {
arthdrlen = 0;
}
#else
printf("\tCompressed file detected and compression support not enabled\n");
arthdrlen = 0;
#endif
} else {
artbase = (char *)base + b + ah.HeadLen;
}
artpos = artbase;
msgId[0] = 0;
newsgroups[0] = 0;
while (arthdrlen > 11) {
int l;
/*
* Scan line
*/
for (l = 0; l < arthdrlen; l++) {
if (artpos[l] == '\n') {
l++;
if (l < arthdrlen && artpos[l] != ' ' && artpos[l] != '\t')
break;
l--;
}
}
if (strncasecmp(artpos, "Message-ID:", 11) == 0) {
diablo_strlcpynl(msgId, artpos + 11, l - 11, sizeof(msgId));
} else if (strncasecmp(artpos, "Newsgroups:", 11) == 0) {
diablo_strlcpynl2(newsgroups, ',', artpos + 11, l - 11, sizeof(newsgroups));
}
arthdrlen -= l;
artpos += l;
}
if (msgId[0]) {
const char *id = MsgId(msgId, NULL);
History h = { 0 };
h.hv = hhash(id);
h.iter = iter;
h.gmt = gmt;
h.exp = 100 + spoolobj;
h.boffset = b;
h.bsize = ah.StoreLen - 1;
headOnly = 0;
if (ah.ArtHdrLen == ah.ArtLen) {
h.exp |= EXPF_HEADONLY;
headOnly = 1;
}
cSize[0] = 0;
if (ah.StoreType & STORETYPE_GZIP) {
h.bsize = ah.ArtLen + ah.HeadLen;
sprintf(cSize, "%d", ah.StoreLen);
}
DoArticle(&h, id, newsgroups, " ", " ", headOnly, "0", cSize);
count++;
} else {
if (VerboseOpt)
printf("No Message-ID for %d,%d\n", b, ah.StoreLen - 1);
}
b += ah.StoreLen;
if (ah.StoreType & STORETYPE_GZIP) {
free(artbase);
b++;
}
}
if (!QuietOpt)
printf("%d entries", count);
}
int
WriteOverview(Connection *conn, ArtNumAss *an, const char *group, const char *xref, const char *art, int artLen, const char *msgid)
{
artno_t artNo = 0;
const char *body;
char *xtmp = NULL;
int xtmpLen = 16 + strlen(conn->co_Auth.dr_VServerDef->vs_ClusterName);
char logbuf[1024];
int err = 0;
/*
* Locate article number assignment
*/
{
ArtNumAss *scan;
for (scan = an; scan; scan = scan->an_Next) {
if (scan->an_GroupName == group) {
artNo = scan->an_ArtNo;
}
xtmpLen += scan->an_GroupLen + 15;
}
}
if (artNo == 0)
return(0);
/*
* XXX We should find some way to aggregate these into one log entry.
*/
snprintf(logbuf, sizeof(logbuf), "%s %s:%lld", msgid, group, artNo);
/*
* Locate start of body (we may have to append our own Xref: line)
*/
{
int l;
int lnl = 1;
for (l = 0; l < artLen; ++l) {
/*
* blank line terminates headers
*/
if (art[l] == '\r' && (l + 1 >= artLen || art[l+1] == '\n')) {
if (lnl)
break;
}
lnl = 0;
if (art[l] == '\n')
lnl = 1;
}
body = art + l;
}
/*
* Write overview record.
*/
{
off_t pos;
OverInfo *ov;
const OverArt *oa;
hash_t hv = hhash(msgid);
int actLen = 0;
int iovLen = 0;
int prealloc_ov, prealloc_oh;
struct iovec iov[3];
if ((ov = GetOverInfo(group)) == NULL) {
logit(LOG_ERR, "Error in GetOverInfo(%s) msgid=%s", group, msgid);
LogCmd(conn, '-', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s - %s%s", conn->co_Auth.dr_Host, "", logbuf);
MBLogPrintf(conn, &conn->co_TMBuf, "400 Error writing data\r\n");
NNTerminate(conn);
return(1);
}
prealloc_ov = (ov->ov_MaxArts * sizeof(OverArt)) / 8;
prealloc_oh = (1024 * ov->ov_Head->oh_DataEntries) / 8;
if (MakeOverHFile(ov, artNo, 1) == NULL) {
logit(LOG_ERR, "Error in MakeOverHFile() msgid=%s", msgid);
LogCmd(conn, '-', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s - %s%s", conn->co_Auth.dr_Host, "", logbuf);
MBLogPrintf(conn, &conn->co_TMBuf, "400 Error writing data\r\n");
NNTerminate(conn);
return(1);
}
hflock(ov->ov_HCache->od_HFd, 0, XLOCK_EX);
pos = lseek(ov->ov_HCache->od_HFd, 0L, 2);
errno = 0;
if (xref) {
iov[0].iov_base = (void *)art;
iov[0].iov_len = artLen + 1;
iovLen = 1;
} else {
ArtNumAss *scan;
int soff;
xtmp = zalloc(&conn->co_MemPool, xtmpLen);
sprintf(xtmp, "Xref: %s", DOpts.ReaderXRefHost);
soff = strlen(xtmp);
for (scan = an; scan; scan = scan->an_Next) {
xtmp[soff++] = ' ';
memcpy(xtmp + soff, scan->an_GroupName, scan->an_GroupLen);
soff += scan->an_GroupLen;
sprintf(xtmp + soff, ":%lld", scan->an_ArtNo);
soff += strlen(xtmp + soff);
}
sprintf(xtmp + soff, "\r\n");
soff += 2;
iov[0].iov_base = (void *)art;
iov[0].iov_len = body - art;
iov[1].iov_base = xtmp;
iov[1].iov_len = soff;
iov[2].iov_base = (void *)body;
iov[2].iov_len = (art + artLen + 1) - body;
iovLen = 3;
}
if (art[0] == 0)
logit(LOG_ERR, "Warning: art[0] is NIL! %s", xtmp);
{
int i;
for (i = 0; i < iovLen; ++i)
actLen += iov[i].iov_len;
}
if (DOpts.ReaderXRefSlaveHost &&
(oa = GetOverArt(ov, artNo, NULL)) != NULL &&
oa->oa_MsgHash.h1 == hv.h1 &&
oa->oa_MsgHash.h2 == hv.h2 &&
OA_ARTNOEQ(artNo, oa->oa_ArtNo)
) {
/*
* We can detect duplicate articles in XRef slave mode. If
* we see one, do not do anything.
*/
LogCmd(conn, 'd', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s d %s%s", conn->co_Auth.dr_Host, "", logbuf);
; /* EMPTY */
} else if (DOpts.ReaderXOverMode == 0) {
/*
* Do not write xover info at all. This mode is not really
* supported by the reader but may eventually be supported
* in 100% nntp-cache mode if/when we develop it.
*/
logit(LOG_INFO, "ReaderXOverMode0 (%lld:%s)", artNo, msgid);
LogCmd(conn, 'm', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s m %s%s", conn->co_Auth.dr_Host, "", logbuf);
; /* EMPTY */
} else if (
DOpts.ReaderXOverMode == 2 ||
(FilePreAllocSpace(ov->ov_HCache->od_HFd, pos,
prealloc_oh, iovLen) == 0 &&
writev(ov->ov_HCache->od_HFd, iov, iovLen) == actLen)
) {
/*
* Our write of the overview data succeeded or we were asked not
* to write out the overview data. Write out the overview
* article record.
*/
OverArt ovart = { 0 };
off_t ovpos = 0;
hflock(ov->ov_OFd, 0, XLOCK_EX);
(void)GetOverArt(ov, artNo, &ovpos);
ovart.oa_ArtNo = OA_ARTNOSET(artNo);
if (DOpts.ReaderXOverMode == 2)
ovart.oa_SeekPos = -1;
else
ovart.oa_SeekPos = pos;
ovart.oa_Bytes = actLen - 1; /* do not include \0 */
ovart.oa_MsgHash = hv;
ovart.oa_TimeRcvd = (int)time(NULL);
ovart.oa_ArtSize = (conn->co_ByteCounter > 0.0 ? conn->co_ByteCounter : conn->co_BytesHeader);
lseek(ov->ov_OFd, ovpos, 0);
FilePreAllocSpace(ov->ov_OFd, ovpos, prealloc_ov, sizeof(ovart));
write(ov->ov_OFd, &ovart, sizeof(ovart));
hflock(ov->ov_OFd, 0, XLOCK_UN);
LogCmd(conn, '+', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s + %s%s", conn->co_Auth.dr_Host, "", logbuf);
} else {
ftruncate(ov->ov_HCache->od_HFd, pos);
logit(LOG_ERR, "error writing overview data file for %s", group);
LogCmd(conn, '-', logbuf);
if (DRIncomingLogPat != NULL)
LogIncoming("%s - %s%s", conn->co_Auth.dr_Host, "", logbuf);
MBLogPrintf(conn, &conn->co_TMBuf, "400 Error writing data\r\n");
NNTerminate(conn);
err = 1;
}
hflock(ov->ov_HCache->od_HFd, 0, XLOCK_UN);
PutOverInfo(ov);
}
if (xtmp)
zfree(&conn->co_MemPool, xtmp, xtmpLen);
return(err);
}
int
DILookup(char *id)
{
hash_t hv;
History h;
int r = 0;
char *p;
if (id[0] == '<' && (p = strchr(id, '>')) != NULL) {
*++p = 0;
hv = hhash(id);
} else if (id[0] == 'D' && id[1] == '.') {
int32 dummy;
char *p = strchr(id, '/');
if (p && p[1] == 'B' && p[2] == '.') {
/*
* dqueue data format
*/
if ((id = strchr(p, '<')) != NULL && strchr(id, '>') != NULL) {
*(strchr(id, '>') + 1) = 0;
hv = hhash(id);
} else {
fprintf(stderr, "argument error: %s\n", id);
exit(1);
}
} else {
/*
* hash code format 1
*/
if (sscanf(id + 2, "%x/%x.%x", &dummy, &hv.h1, &hv.h2) != 3) {
fprintf(stderr, "argument error: %s\n", id);
exit(1);
}
}
} else if (strncmp(id, "DUMP ", 5) == 0) {
if (sscanf(id + 5, "%x.%x", &hv.h1, &hv.h2) != 2) {
fprintf(stderr, "argument error: %s\n", id);
exit(1);
}
} else if (sscanf(id, "%x.%x", &hv.h1, &hv.h2) != 2) {
/*
* hash code format 2
*/
fprintf(stderr, "argument error: %s\n", id);
exit(1);
}
if (HistoryLookupByHash(hv, &h) == 0) {
char tbuf1[64];
char tbuf2[64];
char buf[1024];
if (QuietOpt)
return(r);
{
struct tm *tp;
time_t t;
t = h.gmt * 60;
tp = localtime(&t);
strftime(tbuf1, sizeof(tbuf1), "%d-%b-%Y %H:%M:%S", tp);
if (H_EXPIRED(h.exp)) {
if (h.iter == (unsigned short)-1)
sprintf(tbuf2, "rejected");
else
sprintf(tbuf2, "expired");
} else {
sprintf(tbuf2, "valid");
}
}
if (h.boffset || h.bsize) {
ArticleFileName(buf, sizeof(buf), &h, ARTFILE_FILE_REL);
printf(" [%s hv=%08x.%08x spool=%02x gm=%d ex=%d off=%d len=%d f=%s]"
" GM=(%s) EX=(%s)\n",
buf,
h.hv.h1,
h.hv.h2,
(int)H_SPOOL(h.exp),
(int)h.gmt,
(int)h.exp,
(int)h.boffset,
(int)h.bsize,
((h.exp & EXPF_HEADONLY) ? "H" : ""),
tbuf1,
tbuf2
);
} else {
ArticleFileName(buf, sizeof(buf), &h, ARTFILE_DIR_REL);
printf(" [%s/NOFILE hv=%08x.%08x gm=%d ex=%d f=%s] GM=(%s) EX=(%s) (pre-expired)\n",
buf,
h.hv.h1,
h.hv.h2,
(int)h.gmt,
(int)h.exp,
((h.exp & EXPF_HEADONLY) ? "H" : ""),
tbuf1,
tbuf2
);
}
} else {
printf("Not Found: %s (%08x.%08x)\n", id, hv.h1, hv.h2);
r = 1;
}
return(r);
}