void
FreeOverData(OverData *od)
{
if (od->od_HMapBase) {
xunmap((void *)od->od_HMapBase, od->od_HMapBytes);
od->od_HMapBase = NULL;
}
if (od->od_HFd >= 0) {
close(od->od_HFd);
od->od_HFd = -1;
}
zfree(&SysMemPool, od, sizeof(OverData));
}
void
ScanSpoolFile(char *fpath, int gmt, int iter, uint16 spoolobj)
{
int fd;
char *base;
int bytes;
struct stat st;
if (VerboseOpt)
printf(" Scanning file: %s\n", fpath);
errno = 0;
if ((fd = open(fpath, O_RDONLY)) < 0) {
printf(" %s\t%s\n", fpath, strerror(errno));
return;
}
if (fstat(fd, &st) < 0) {
printf(" %s\t%s\n", fpath, strerror(errno));
close(fd);
return;
}
bytes = st.st_size;
base = xmap(NULL, bytes, PROT_READ, MAP_SHARED, fd, 0);
if (base == NULL) {
printf(" %s\t%s\n", fpath, strerror(errno));
close(fd);
return;
}
if (!QuietOpt)
printf(" %s: ", fpath);
if (bytes > 2 && (uint8)*base == (uint8)STORE_MAGIC1 &&
(uint8)*(base + 1) == (uint8)STORE_MAGIC2)
ScanSpoolFileMap(base, bytes, gmt, iter, fpath, spoolobj, fd);
else
ScanSpoolFileMapOld(base, bytes, gmt, iter, fpath, spoolobj);
if (!QuietOpt)
printf("\n");
xunmap(base, bytes);
close(fd);
}
void
FreeOverInfo(OverInfo *ov)
{
OverData *od;
while ((od = ov->ov_HData) != NULL) {
ov->ov_HData = od->od_Next;
FreeOverData(od);
}
if (ov->ov_Head)
xunmap((void *)ov->ov_Head, ov->ov_Size);
if (ov->ov_OFd >= 0) {
/*
* remove shared lock and close
*/
hflock(ov->ov_OFd, 4, XLOCK_UN);
close(ov->ov_OFd);
}
zfreeStr(&SysMemPool, &ov->ov_Group);
bzero(ov, sizeof(OverInfo));
zfree(&SysMemPool, ov, sizeof(OverInfo));
}
int
DumpArticle(char *fname, History *h, const char *msgid)
{
int fd;
int rv = 0;
int wireFormat = 0;
int artSize = 0;
int headLen;
int oldFormat = 0;
int compressedFormat = 0;
SpoolArtHdr ah;
if ((fd = open(fname, O_RDONLY)) >= 0) {
char *base = NULL;
const char *ptr;
int extra = (h->boffset == 0) ? 0 : 1;
errno = 0;
if (ShowFileHeader) {
rv = DumpFileHeader(fd, h->boffset);
close(fd);
return(rv);
}
if (MapArticle(fd, fname, &base, h, &extra, &artSize, &compressedFormat) != 0)
return(1);
/*
* check for prior terminating zero, article body does not
* begin with a null, and post terminating zero
*/
ptr = base;
if (rv == 0 && extra) {
if (*ptr != 0) {
fprintf(LogFo, " missingPreNul");
rv = 1;
}
++ptr;
}
if (rv == 0 && ptr[0] == 0) {
fprintf(LogFo, " nullArtBody");
rv = 1;
}
if (rv == 0 && ptr[artSize] != 0) {
fprintf(LogFo, " missingPostNul");
rv = 1;
}
bcopy(ptr, &ah, sizeof(ah));
if (DebugOpt) {
printf("magic1=%x magic2=%x\n", (uint8)ah.Magic1,
(uint8)ah.Magic2);
}
if ((uint8)ah.Magic1 == (uint8)STORE_MAGIC1 &&
(uint8)ah.Magic2 == (uint8)STORE_MAGIC2) {
headLen = (uint32)ah.ArtHdrLen;
artSize -= (uint8)ah.HeadLen;
ptr += (uint8)ah.HeadLen;
wireFormat = 0;
if ((uint8)ah.StoreType & STORETYPE_WIRE)
wireFormat = 1;
} else {
headLen = artSize;
wireFormat = 0;
oldFormat = 1;
}
/*
* Locate Message-ID header and test
*/
if (rv == 0 && msgid) {
const char *l;
int haveMsgId = 0;
for (l = ptr - 1; l < ptr + headLen; l = strchr(l, '\n')) {
if (l == NULL) {
rv = 1;
break;
}
++l;
if (strncasecmp(l, "Message-ID:", 11) == 0) {
int i;
haveMsgId = 1;
l += 11;
while (*l && *l != '<' && *l != '\n')
++l;
for (i = 0; l[i] && l[i] != '>' && l[i] != '\n' &&
l[i] != '\r'; ++i) {
if (msgid[i] != l[i])
rv = 1;
}
if (msgid[i] != l[i])
rv = 1;
}
if (!wireFormat && l[0] == '\n') /* end of headers */
break;
else if (wireFormat && l[0] == '\r' && l[1] == '\n')
break;
}
if (rv) {
fprintf(LogFo, " messageID-MisMatch");
} else if (haveMsgId == 0) {
fprintf(LogFo, " missing-MessageID");
rv = 1;
}
}
if (rv == 0)
fprintf(LogFo, "%sOK", oldFormat ? "(old spool) " : "");
fprintf(LogFo, "\n");
if (rv == 0 && VerifyOnly == 0) {
int i;
int lastNl = 1;
int lineLen = 0;
fflush(LogFo);
fflush(stdout);
if (HeadOnly) {
for (i = 0; i < headLen; ++i) {
if (StripCR && wireFormat && i > 0 &&
ptr[i-1] == '\r' && ptr[i] == '\n') {
if (!QuietOpt) {
write(1, ptr + i - lineLen, lineLen - 1);
write(1, "\n", 1);
}
if (lineLen == 1)
break;
lineLen = 0;
} else if (wireFormat && i > 0 &&
ptr[i-1] == '\r' && ptr[i] == '\n') {
if (lineLen == 1)
break;
lastNl = 1;
lineLen = 0;
} else if (ptr[i] == '\n') {
if (lastNl)
break;
lastNl = 1;
} else {
lastNl = 0;
lineLen++;
}
}
if ((!StripCR || !wireFormat) && !QuietOpt)
write(1, ptr, i);
} else {
if (StripCR && wireFormat) {
for (i = 0; i < artSize; ++i) {
if (i > 0 && ptr[i-1] == '\r' && ptr[i] == '\n') {
if (!QuietOpt) {
write(1, ptr + i - lineLen, lineLen - 1);
write(1, "\n", 1);
}
lineLen = 0;
} else {
lineLen++;
}
}
} else if (!QuietOpt) {
write(1, ptr, artSize);
}
}
fflush(stdout);
}
if (base != NULL) {
if (compressedFormat)
free(base);
else
xunmap((void *)base, h->bsize + extra + 1);
}
if (fd != -1 && !compressedFormat)
close(fd);
} else {
fprintf(LogFo, "Unable to open %s\n", fname);
rv = 1;
}
return(rv);
}
int
main(int argc, char **argv) {
extern char *optarg;
extern int optind;
char *progname = *argv;
char *HistoryFile;
int SleepTime = 5;
int Opened = 0;
char ch;
struct stat st;
int PrevIno = -1;
int Force = 0;
char *map = NULL;
off_t mapsize = 0;
int All = 0;
optind = 1;
while ((ch = getopt(argc, argv, "afsV")) != -1) {
switch(ch) {
case 'a':
All = 1;
break;
case 'f':
Force = 1;
break;
case 's':
SleepTime = strtol(optarg, NULL, 0);
break;
case 'V':
PrintVersion();
break;
default:
usage(argv[0]);
}
}
argv += optind;
if (*argv == NULL)
usage(progname);
HistoryFile = *argv;
if (!Force && geteuid() != 0) {
printf("This daemon must be run as root due to the use of mlock()\n");
printf("which is only allowed to be executed by the root user\n");
exit(1);
}
while (1) {
if (!Opened) {
if (stat(HistoryFile, &st) == 0)
PrevIno = st.st_ino;
if (All) {
int fd;
mapsize = st.st_size;
fd = open(HistoryFile, O_RDONLY);
if (fd == -1) {
perror("history open");
exit(1);
}
map = xmap(NULL, mapsize, PROT_READ, MAP_SHARED, fd, 0);
close(fd);
mlock(map, mapsize);
} else {
HistoryOpen(HistoryFile, HGF_MLOCK);
}
Opened = 1;
}
sleep(SleepTime);
if (stat(HistoryFile, &st) != 0 || st.st_ino != PrevIno ||
(All && st.st_size != mapsize)) {
if (All && map != NULL) {
munlock(map, mapsize);
xunmap((void *)map, mapsize);
} else {
HistoryClose();
}
Opened = 0;
printf("New history\n");
}
}
}
const char *
GetOverRecord(OverInfo *ov, artno_t artno, int *plen, int *alen, TimeRestrict *tr, int *TimeRcvd)
{
int hvpos;
int xpos;
int xsize;
const OverArt *oa;
OverData *od;
oa = GetOverArt(ov, artno, NULL);
if (oa == NULL || ! OA_ARTNOEQ(artno, oa->oa_ArtNo) || oa->oa_Bytes > OVER_HMAPSIZE / 2) {
if (plen)
*plen = 0;
return(NULL);
}
if (ov->ov_LimitSecs > 0) {
int dt = (int)(CurTime.tv_sec - oa->oa_TimeRcvd);
if (dt > ov->ov_LimitSecs) {
if (plen)
*plen = 0;
return(NULL);
}
}
if (tr && tr->tr_Time > oa->oa_TimeRcvd)
return(NULL);
if (TimeRcvd)
*TimeRcvd = oa->oa_TimeRcvd;
if (alen)
*alen = oa->oa_ArtSize;
if (plen == NULL)
return((const char *)1);
if (oa->oa_SeekPos == -1)
return(NULL);
if ((od = MakeOverHFile(ov, artno, 0)) == NULL)
return(NULL);
/*
* hvpos / oa->oa_Bytes. Include the guard character(s) in our
* calculations.
*/
hvpos = oa->oa_SeekPos;
xsize = oa->oa_Bytes + 1;
if ((xpos = hvpos) != 0) {
--xpos;
++xsize;
}
if (
od->od_HMapBase == NULL ||
xpos < od->od_HMapPos ||
xpos + xsize > od->od_HMapPos + od->od_HMapBytes
) {
struct stat st;
if (od->od_HMapBase) {
xunmap((void *)od->od_HMapBase, od->od_HMapBytes);
od->od_HMapBase = NULL;
od->od_HMapBytes = 0;
od->od_HMapPos = 0;
}
st.st_size = 0;
fstat(od->od_HFd, &st);
/*
* Make sure the file is big enough to map requested header. It
* is possible for it to not be.
*/
if (xpos + xsize > st.st_size)
return(NULL);
od->od_HMapPos = xpos & ~(HMAPALIGN-1);
od->od_HMapBytes = OVER_HMAPSIZE;
if (od->od_HMapBytes + od->od_HMapPos > st.st_size)
od->od_HMapBytes = st.st_size - od->od_HMapPos;
od->od_HMapBase = xmap(NULL, od->od_HMapBytes, PROT_READ, MAP_SHARED, od->od_HFd, od->od_HMapPos);
if (od->od_HMapBase == NULL) {
logit(LOG_CRIT, "mmap() failed B %s", strerror(errno));
exit(1);
}
}
/*
* Return base of record, length in *plen. But check for corruption...
* if the overview starts with a nul we have a problem.
*/
*plen = oa->oa_Bytes;
{
const char *r = od->od_HMapBase + hvpos - od->od_HMapPos;
if (*r == 0)
return(NULL);
if (xpos < hvpos && r[-1] != 0) {
logit(LOG_ERR, "corrupt overview entry for %s:%lld", ov->ov_Group, artno);
return(NULL);
}
if (r[oa->oa_Bytes] != 0) {
logit(LOG_ERR, "corrupt overview entry for %s:%lld", ov->ov_Group, artno);
return(NULL);
}
return(r);
}
}
OverData *
MakeOverHFile(OverInfo *ov, artno_t artNo, int create)
{
artno_t artBase = artNo & ~ov->ov_DataEntryMask;
OverData **pod;
OverData *od;
int count = 0;
int compressed = 0;
int hfd, tmpfd, zfd, gzflags;
unsigned char *zmap, *tmpmap, *zpos, tmpfile[256];
struct stat st;
unsigned long zlen;
z_stream z;
int code;
if (create)
create = O_CREAT;
if (ov->ov_HCache && artBase == ov->ov_HCache->od_ArtBase)
return(ov->ov_HCache);
for (pod = &ov->ov_HData; (od = *pod) != NULL; pod = &od->od_Next) {
if (artBase == od->od_ArtBase)
break;
++count;
}
if (od == NULL) {
const char *gfname = GFName(ov->ov_Group, GRPFTYPE_DATA, artBase, 1,
ov->ov_Iter, &DOpts.ReaderGroupHashMethod);
*pod = od = zalloc(&SysMemPool, sizeof(OverData));
errno = 0;
hfd = xopen(O_RDWR|create, 0644, "%s/%s", MyGroupHome, gfname);
if (hfd < 0 && ! create) {
zfd = xopen(O_RDONLY, 0644, "%s/%s.gz", MyGroupHome, gfname);
if (! (zfd < 0)) {
compressed++;
snprintf(tmpfile, sizeof(tmpfile), "/tmp/dr-gzover.XXXXXXXX");
tmpfd = mkstemp(tmpfile);
if (! (tmpfd < 0)) {
st.st_size = 0;
fstat(zfd, &st);
zmap = xmap(NULL, st.st_size, PROT_READ, MAP_SHARED, zfd, 0);
if (! zmap) {
logit(LOG_ERR, "Unable to xmap for gzcat %s/%s: %s",
MyGroupHome,
gfname,
strerror(errno)
);
} else {
zpos = zmap + st.st_size - 4;
zlen = ((unsigned)zpos[0] & 0xff) |
((unsigned)zpos[1] & 0xff) << 8 |
((unsigned)zpos[2] & 0xff) << 16 |
((unsigned)zpos[3] & 0xff) << 24;
if (zlen < 256 || zlen > (64 * 1048576)) {
logit(LOG_ERR, "Bad zlen %d for gzcat %s/%s",
zlen,
MyGroupHome,
gfname
);
xunmap((void *)zmap, st.st_size);
hfd = -1;
close(tmpfd);
tmpfd = -1;
} else {
ftruncate(tmpfd, zlen);
tmpmap = xmap(NULL, zlen, PROT_READ|PROT_WRITE, MAP_SHARED, tmpfd, 0);
if (! tmpmap) {
logit(LOG_ERR, "Unable to xmap for gztmp /tmp/%s: %s",
MyGroupHome,
gfname,
strerror(errno)
);
xunmap((void *)zmap, st.st_size);
hfd = -1;
close(tmpfd);
tmpfd = -1;
} else {
// handle gzip headers
zpos = zmap;
if (zpos[0] != GZ_MAGIC0 || zpos[1] != GZ_MAGIC1 || zpos[2] != Z_DEFLATED) {
logit(LOG_ERR, "gzip header error (%d, %d, %d) for gzcat %s/%s: %s",
zpos[0], zpos[1], zpos[2],
MyGroupHome,
gfname,
z.msg
);
}
zpos += 3;
gzflags = *zpos;
zpos += 7;
if ((gzflags & GZ_EXTRA)) {
zpos += zpos[0] + (zpos[1] << 8);
}
if ((gzflags & GZ_ORIGNAME)) {
for ( ; *zpos; zpos++) {
}
zpos++;
}
if ((gzflags & GZ_COMMENT)) {
for ( ; *zpos; zpos++) {
}
zpos++;
}
if ((gzflags & GZ_HEADCRC)) {
zpos += 2;
}
// begin uncompress
bzero(&z, sizeof(z));
z.next_in = zpos;
z.avail_in = st.st_size - (zpos - zmap);
z.next_out = tmpmap;
z.avail_out = zlen;
// z.zalloc = zalloc;
// z.zfree = zfree;
// z.opaque = &SysMemPool;
inflateInit2(&z, -MAX_WBITS);
code = inflate(&z, Z_FINISH);
inflateEnd(&z);
if (code != Z_STREAM_END) {
logit(LOG_ERR, "inflate error (%i) for gzcat %s/%s: %s",
code,
MyGroupHome,
gfname,
z.msg
);
xunmap((void *)zmap, st.st_size);
xunmap((void *)tmpmap, zlen);
hfd = -1;
close(tmpfd);
tmpfd = -1;
} else {
xunmap((void *)zmap, st.st_size);
hfd = tmpfd;
// od->od_HMapBase = tmpmap;
// od->od_HMapBytes = zlen;
xunmap((void *)tmpmap, zlen);
}
}
}
}
close(zfd);
if (unlink(tmpfile) < 0) {
logit(LOG_ERR, "Unable to remove gztmp %s: %s",
tmpfile,
strerror(errno)
);
}
} else {
logit(LOG_ERR, "Unable to open/create gztmp %s: %s",
tmpfile,
strerror(errno)
);
close(zfd);
}
}
}
od->od_HFd = hfd;
if (od->od_HFd < 0) {
if (create) {
logit(LOG_ERR, "Unable to open/create %s/%s: %s",
MyGroupHome,
gfname,
strerror(errno)
);
}
FreeOverData(od);
*pod = od = NULL;
} else {
od->od_ArtBase = artBase;
if (count > DOpts.ReaderThreads) {
OverData *t = ov->ov_HData;
ov->ov_HData = t->od_Next;
FreeOverData(t);
}
}
}
ov->ov_HCache = od;
return(od);
}
void
OutputOverRange(OverInfo *ov, Connection *conn)
{
int hvpos;
int xpos=0;
int xsize=0;
int nart=0;
const OverArt *oa;
OverData *od;
artno_t artBase = conn->co_ListBegNo & ~ov->ov_DataEntryMask;
artno_t artend = conn->co_ListEndNo;
TimeRestrict *tr = NULL;
if (conn->co_ArtMode == COM_NEWNEWS)
tr = &conn->co_TimeRestrict;
/* one datafile at a time */
if (artend>artBase+ov->ov_DataEntryMask) {
artend = artBase+ov->ov_DataEntryMask;
}
/* process mmap needs */
if ((conn->co_ListBegNo>artBase) || (artend<artBase+ov->ov_DataEntryMask)) {
nart = ProcessOverMmapNeed(ov, conn, tr, artend, OVER_HMAPSIZE, &xpos, &xsize);
if (nart==0) return;
if (xpos) xpos--;
xsize -= xpos-1;
}
/* mmaping datafile */
if ((od = MakeOverHFile(ov, artBase, 0)) == NULL) {
conn->co_ListBegNo = artend+1;
return;
}
if (
od->od_HMapBase == NULL ||
nart == 0 ||
xpos < od->od_HMapPos ||
xpos + xsize > od->od_HMapPos + od->od_HMapBytes
) {
struct stat st;
int advise=XADV_WILLNEED;
if (od->od_HMapBase) {
xunmap((void *)od->od_HMapBase, od->od_HMapBytes);
od->od_HMapBase = NULL;
od->od_HMapBytes = 0;
od->od_HMapPos = 0;
}
st.st_size = 0;
fstat(od->od_HFd, &st);
/*
* Make sure the file is big enough to map requested header. It
* is possible for it to not be.
*/
if (!st.st_size) {
if (xsize) {
logit(LOG_CRIT, "Group %s data file is empty (%i/%i)", conn->co_GroupName, st.st_size, xsize);
}
conn->co_ListBegNo = artend+1;
return;
}
if (xpos > st.st_size) {
logit(LOG_CRIT, "Group %s data file is too small to be mmapped (%i/%i)", conn->co_GroupName, st.st_size, xpos);
conn->co_ListBegNo = artend+1;
return;
}
if (nart==0) {
if (st.st_size>OVER_HMAPSIZE) {
nart = ProcessOverMmapNeed(ov, conn, tr, artend, OVER_HMAPSIZE, &xpos, &xsize);
if (nart==0) return;
if (xpos) xpos--;
xsize -= xpos-1;
} else {
xpos = 0;
xsize = st.st_size;
}
}
od->od_HMapPos = xpos;
if (xpos + xsize > st.st_size) {
/* check if first article is inside the file */
oa = GetOverArt(ov, conn->co_ListBegNo, NULL);
if (oa->oa_SeekPos + oa->oa_Bytes > st.st_size) {
logit(LOG_CRIT, "Group %s data file is too small to contain header #%lld (%i+%i>%i)", conn->co_GroupName, conn->co_ListBegNo, oa->oa_SeekPos, oa->oa_Bytes, st.st_size);
conn->co_ListBegNo++;
return;
}
logit(LOG_CRIT, "Group %s data file is too small to be fully mmapped (%i+%i>%i)", conn->co_GroupName, xpos, xsize, st.st_size);
od->od_HMapBytes = st.st_size - xpos;
} else {
od->od_HMapBytes= xsize;
}
od->od_HMapBase = xmap(NULL, od->od_HMapBytes, PROT_READ, MAP_SHARED, od->od_HFd, od->od_HMapPos);
if (od->od_HMapBase == NULL) {
logit(LOG_CRIT, "mmap() failed C (%s) group %s (%lld:%lld@%i:%i)", strerror(errno), conn->co_GroupName, conn->co_ListBegNo, artend, od->od_HMapPos, od->od_HMapBytes);
exit(1);
}
xadvise(od->od_HMapBase, od->od_HMapBytes, advise);
}
for( ; conn->co_ListBegNo <= artend ; conn->co_ListBegNo++) {
oa = GetOverArt(ov, conn->co_ListBegNo, NULL);
if (oa==NULL || ! OA_ARTNOEQ(conn->co_ListBegNo, oa->oa_ArtNo)) continue;
if (tr && tr->tr_Time > oa->oa_TimeRcvd) continue;
if (oa->oa_SeekPos == -1) continue;
/* check mmap */
if ( (oa->oa_SeekPos<od->od_HMapPos)
|| (oa->oa_SeekPos+oa->oa_Bytes-od->od_HMapPos > od->od_HMapBytes)
) return;
/*
* hvpos / oa->oa_Bytes. Include the guard character(s) in our
* calculations.
*/
hvpos = oa->oa_SeekPos;
xsize = oa->oa_Bytes + 1;
if ((xpos = hvpos) != 0) {
--xpos;
++xsize;
}
/*
* Return base of record, length in *plen. But check for corruption...
* if the overview starts with a nul we have a problem.
*/
{
const char *r = od->od_HMapBase + hvpos - od->od_HMapPos;
if (*r == 0)
continue;
if (xpos < hvpos && r[-1] != 0) {
logit(LOG_ERR, "corrupt overview entry for %s:%lld", ov->ov_Group, conn->co_ListBegNo);
continue;
}
if (r[oa->oa_Bytes] != 0) {
logit(LOG_ERR, "corrupt overview entry for %s:%lld", ov->ov_Group, conn->co_ListBegNo);
continue;
}
OutputOverview(conn, r, oa->oa_Bytes, oa->oa_ArtSize);
}
}
}