static int TDBread(tdb_t tdb) {
rewind(tdb->file);
if (ReadOnly(tdb->file)) {
tdb->gzfile = GZdopen(dup(fileno(tdb->file)), "r");
ParseFile(tdb->gzfile, tdb, 1000000);
GZclose(tdb->gzfile);
tdb->gzfile = NULL;
}
else {
int i;
tdb->gzfile = GZdopen(dup(fileno(tdb->file)),"r");
ParseFile(tdb->gzfile, tdb, 1000000);
GZclose(tdb->gzfile);
tdb->gzfile = GZdopen(dup(fileno(tdb->file)), "w");
for (i=0;i<tdb->count;i++) {
int new;
TDBput(tdb, TDBget(tdb, i), &new);
}
}
fclose(tdb->file);
tdb->file = NULL;
return 0;
}
int gunzipFilterX(FILE *in,FILE *out,SyncF syncf,void *sp,int si){
gzFile gz;
int rcc;
int wcc;
int werr;
CStr(buf,1024*8);
int size;
double Start = Time();
const char *em;
int en;
int ef;
int ready;
int rd;
int eof = 0;
int nonblock;
int serrno = 0;
int ibz = sizeof(buf);
int gi;
int fd = -1;
inlen = 0;
errno = 0;
fd = dup(fileno(in));
if( isWindows() ){
int pollPipe(int,int);
nonblock = 0;
ready = fPollIn(in,10*1000);
gz = GZdopen(fd,"r");
if( gz == 0 )
syslog_ERROR("##gunzipFilter[%d/%d] gz=%X ready=%d/%d\n",
fd,fileno(in),p2i(gz),ready,pollPipe(fd,1));
}else{
/*
* to make smooth streaming of data relayed on narrow network
* apply NBIO to gzopen() which will do fread() at the start.
* applying NBIO also to gzread() seems to break the gzip.
*/
/*
setNonblockingIO(fileno(in),1);
*/
setNonblockingIO(fd,1);
nonblock = 1;
ready = fPollIn(in,10*1000);
if( ready == 0 ){
fprintf(stderr,"----[%d] gunzipFilter: ready[%d]=%d\n",
getpid(),fileno(in),ready);
}
/*
gz = GZdopen(fd = dup(fileno(in)),"r");
*/
gz = GZdopen(fd,"r");
if( DGzlibVer == 0 )
{
/*
setNonblockingIO(fileno(in),0);
*/
setNonblockingIO(fd,0);
nonblock = 0;
}
}
if( syncf != 0 ){
syslog_ERROR("--- gunzipFX SYNC %X(%X,%d)\n",xp2i(syncf),p2i(sp),si);
(*syncf)(sp,si);
}
ibz = 1024;
//ibz = 256;
size = 0;
/*
if( gz = GZdopen(dup(fileno(in)),"r") ){
*/
if( gz ){
LOGX_gunzip++;
setCloseOnFork("GUNZIPstart",fd);
em = gzerror(gz,&en);
/*
while( 0 < (rcc = gzread(gz,buf,sizeof(buf))) ){
*/
for( gi = 0;; gi++ ){
if( gotsigTERM("gunzip gi=%d em=%X",gi,p2i(em)) ){
if( numthreads() ){
if( em ){
putfLog("thread-gunzip gi=%d _exit() em=(%s)",gi,em?em:"");
_exit(0);
}
thread_exit(0);
}
break;
}
if( nonblock ){
if( 0 < gi ){
/*
setNonblockingIO(fileno(in),0);
*/
setNonblockingIO(fd,0);
nonblock = 0;
}
}
/*
if( 0 < size && inputReady(fileno(in),NULL) == 0 ){
*/
/*
if( 0 < size && inputReady(fd,NULL) == 0 ){
*/
if( eof == 0 )
if( 0 < size )
if( ready = inputReady(fd,&rd) ){
if( ready == 2 ){ /* both PS_IN and PS_PRI */
eof = 1;
}
}else{
//fprintf(stderr,"[%d] -- gzread#%d %d / %d FLUSH\n",getpid(),gi,rcc,size);
fflush(out);
}
ready = fPollIn(in,10*1000);
errno = 0;
rcc = gzread(gz,buf,QVSSize(buf,ibz));
serrno = errno;
if( rcc <= 0 ){
break;
}
//fprintf(stderr,"[%d] -- gzread %d / %d\n",getpid(),rcc,size);
wcc =
fwrite(buf,1,rcc,out);
/* this fflush seems significant */
werr =
fflush(out);
if( wcc < rcc || werr || ferror(out) || gotSIGPIPE() ){
porting_dbg("+++EPIPE[%d] gunzip fwrite() %d/%d err=%d/%d %d SIG*%d",fileno(out),wcc,rcc,werr,ferror(out),size,gotSIGPIPE());
break;
}
size += rcc;
if( size < sizeof(buf) ){
fflush(out);
}else{
ibz = sizeof(buf);
}
}
fflush(out);
if( rcc < 0 || size == 0 ){
em = gzerror(gz,&en);
ef = gzeof(gz);
if( en == -1 /* see errno */ && serrno == 0 ){
/* no error */
}else{
daemonlog("F","FATAL: gzread(%d)=%d/%d eof=%d %d %s %d\n",
fd,rcc,size,ef,en,em,serrno);
porting_dbg("FATAL: gzread(%d)=%d/%d eof=%d %d %s",
fd,rcc,size,ef,en,em);
if( lTHREAD() )
fprintf(stderr,"--[%d]gzread(%d)=%d/%d eof=%d %d %s\n",
getpid(),fd,rcc,size,ef,en,em);
}
}
clearCloseOnFork("GUNZIPend",fd);
GZclose(gz);
if( isWindowsCE() || lMULTIST() ){
/* duplicated close of fd is harmful */
}else
if( isWindows() ) close(fd);
fseek(out,0,0);
syslog_DEBUG("(%f)gunzipFilter -> %d\n",Time()-Start,size);
if( lTHREAD() )
if( 0 < inlen )
syslog_ERROR("###GUNZIP filter %d/%d\n",inlen,size);
return size;
}
return 0;
}
int gzipFilterX(FILE *in,FILE *out,SyncXF syncf,void *sp,int si){
gzFile gz;
int len,rcc;
CStr(buf,1024*8);
int size;
int gsize;
int wcc;
int bcc = 0;
double Start = Time();
double Prevf = 0;
int ibz = sizeof(buf);
int gi;
int fd = -1;
int ofd = fileno(out);
int xfd;
int zerr = 0;
/*
int rready = -1;
*/
errno = 0;
fd = dup(fileno(out));
if( fd < 0 ){
syslog_ERROR("--gzipFilter[%d]<-[%d] errno=%d\n",fd,ofd,errno);
return -1;
}
/*
if( 0 <= GZIPready )
rready = dup(GZIPready);
*/
len = 0;
/*
if( gz = GZdopen(dup(fileno(out)),"w") ){
*/
if( file_isSOCKET(ofd) || file_ISSOCK(ofd) )
if( !IsConnected(ofd,NULL) || !IsAlive(ofd) ){
fprintf(stderr,"[%d.%X] gzip DISCONN\n",getpid(),getthreadid());
fprintf(stderr,"[%d.%X] gzip DISCONN fd[%d] con=%d isSOCK=%d,%d,%d\n",
getpid(),getthreadid(),ofd,IsConnected(ofd,NULL),
file_isSOCKET(ofd),file_ISSOCK(ofd),file_issock(ofd));
sendsync(rready,1);
close(fd);
return -1;
}
gz = GZdopen(fd,"w");
if( file_isSOCKET(ofd) || file_ISSOCK(ofd) )
if( !IsConnected(ofd,NULL) || !IsAlive(ofd) ){
fprintf(stderr,"[%d.%X] gzip DISCONN gx=%d\n",getpid(),getthreadid(),p2i(gz));
fprintf(stderr,"[%d.%X] gzip DISCONN fd[%d] con=%d isSOCK=%d,%d,%d\n",
getpid(),getthreadid(),ofd,IsConnected(ofd,NULL),
file_isSOCKET(ofd),file_ISSOCK(ofd),file_issock(ofd));
close(fd);
sendsync(rready,2);
close(fd);
return -1;
}
if( gz ){
LOGX_gzip++;
if( Gzip_NoFlush ){
GZDBG(stderr,"-- %X gzip flush disabled(%d)\n",
TID,Gzip_NoFlush);
}
Prevf = Time();
sendsync(rready,0);
setCloseOnFork("GZIPstart",fd);
/*
while( rcc = fread(buf,1,sizeof(buf),in) ){
*/
for( gi = 0;; gi++ ){
if( gotsigTERM("gzip gi=%d",gi) ){
if( numthreads() && !ismainthread() ){
thread_exit(0);
}
break;
}
if( !Gzip_NoFlush )
if( bcc )
if( 0 < len && finputReady(in,NULL) == 0 ){
zerr =
gzflush(gz,Z_SYNC_FLUSH);
if( zerr ){
porting_dbg("+++EPIPE[%d] gzflush() zerr=%d %d SIG*%d",fd,zerr,len,gotSIGPIPE());
}
bcc = 0;
}
if( lSINGLEP() ) /* could be generic */
{
if( 0 < len )
if( !Gzip_NoFlush
|| 4 < gi && 5 < Time()-Prevf
){
GZDBG(stderr,"-- %X gzip flush %d(%f) %d/%d\n",
TID,Gzip_NoFlush,Time()-Start,len,gi);
Prevf = Time();
zerr = gzflush(gz,Z_SYNC_FLUSH);
bcc = 0;
if( zerr ){
GZDBG(stderr,"-- %X gzip gzflush()%d err=%d\n",
TID,len,zerr);
break;
}
}
}
/*
rcc = fread(buf,1,sizeof(buf),in);
*/
rcc = xread(in,AVStr(buf),QVSSize(buf,ibz));
if( rcc <= 0 ){
break;
}
wcc =
gzwrite(gz,buf,rcc);
//fprintf(stderr,"[%d] Gzwrite %d/%d / %d\n",getpid(),wcc,rcc,len);
if( wcc <= 0 ){
porting_dbg("+++EPIPE[%d] gzwrite() %d/%d %d SIG*%d",fd,wcc,rcc,len,gotSIGPIPE());
fprintf(stderr,"[%d] Gzwrite %d/%d / %d\n",getpid(),wcc,rcc,len);
break;
}
if( wcc != rcc ){
syslog_ERROR("gzwrite %d/%d\n",wcc,rcc);
}
if( 0 < wcc ){
bcc += wcc;
}
if( sizeof(buf) <= len ){
ibz = sizeof(buf);
}
if( !Gzip_NoFlush )
if( bcc )
if( sizeof(buf) <= bcc || len < 16*1024 ){
zerr =
gzflush(gz,Z_SYNC_FLUSH);
bcc = 0;
}
if( zerr || gotSIGPIPE() ){
porting_dbg("+++EPIPE[%d] gzflush() zerr=%d %d SIG*%d",fd,zerr,len,gotSIGPIPE());
break;
}
len += rcc;
}
if( len == 0 ){
const char *em;
int en;
int ef;
em = gzerror(gz,&en);
ef = gzeof(gz);
if( en == -1 /* see errno */ && errno == 0 ){
/* no error */
}else{
daemonlog("F","FATAL: gzwrite(%d)=%d/%d eof=%d %d %s\n",
fd,len,bcc,ef,en,em);
porting_dbg("FATAL: gzwrite(%d)=%d/%d eof=%d %d %s",
fd,len,bcc,ef,en,em);
}
}
clearCloseOnFork("GZIPend",fd);
gzflush(gz,Z_SYNC_FLUSH);
xfd = dup(fd);
gsize = GZtell(gz);
GZclose(gz);
if( isWindowsCE() || lMULTIST() ){
/* duplicated close of fd is harmful */
}else
if( isWindows() ) close(fd); /* to clear osf-handle mapping */
Lseek(xfd,0,2);
size = Lseek(xfd,0,1);
Lseek(xfd,0,0);
close(xfd);
syslog_DEBUG("(%f)gzipFilter %d -> %d / %d\n",Time()-Start,
len,gsize,size);
return len;
}
sendsync(rready,3);
close(fd);
return 0;
}
int TDBdestroy(tdb_t tdb) {
ATindexedSetDestroy(tdb->db);
if (tdb->gzfile) GZclose(tdb->gzfile);
free(tdb);
return 0;
}
