static int TDBread(tdb_t tdb) { rewind(tdb->file); if (ReadOnly(tdb->file)) { tdb->gzfile = GZdopen(dup(fileno(tdb->file)), "r"); ParseFile(tdb->gzfile, tdb, 1000000); GZclose(tdb->gzfile); tdb->gzfile = NULL; } else { int i; tdb->gzfile = GZdopen(dup(fileno(tdb->file)),"r"); ParseFile(tdb->gzfile, tdb, 1000000); GZclose(tdb->gzfile); tdb->gzfile = GZdopen(dup(fileno(tdb->file)), "w"); for (i=0;i<tdb->count;i++) { int new; TDBput(tdb, TDBget(tdb, i), &new); } } fclose(tdb->file); tdb->file = NULL; return 0; }
int gunzipFilterX(FILE *in,FILE *out,SyncF syncf,void *sp,int si){ gzFile gz; int rcc; int wcc; int werr; CStr(buf,1024*8); int size; double Start = Time(); const char *em; int en; int ef; int ready; int rd; int eof = 0; int nonblock; int serrno = 0; int ibz = sizeof(buf); int gi; int fd = -1; inlen = 0; errno = 0; fd = dup(fileno(in)); if( isWindows() ){ int pollPipe(int,int); nonblock = 0; ready = fPollIn(in,10*1000); gz = GZdopen(fd,"r"); if( gz == 0 ) syslog_ERROR("##gunzipFilter[%d/%d] gz=%X ready=%d/%d\n", fd,fileno(in),p2i(gz),ready,pollPipe(fd,1)); }else{ /* * to make smooth streaming of data relayed on narrow network * apply NBIO to gzopen() which will do fread() at the start. * applying NBIO also to gzread() seems to break the gzip. */ /* setNonblockingIO(fileno(in),1); */ setNonblockingIO(fd,1); nonblock = 1; ready = fPollIn(in,10*1000); if( ready == 0 ){ fprintf(stderr,"----[%d] gunzipFilter: ready[%d]=%d\n", getpid(),fileno(in),ready); } /* gz = GZdopen(fd = dup(fileno(in)),"r"); */ gz = GZdopen(fd,"r"); if( DGzlibVer == 0 ) { /* setNonblockingIO(fileno(in),0); */ setNonblockingIO(fd,0); nonblock = 0; } } if( syncf != 0 ){ syslog_ERROR("--- gunzipFX SYNC %X(%X,%d)\n",xp2i(syncf),p2i(sp),si); (*syncf)(sp,si); } ibz = 1024; //ibz = 256; size = 0; /* if( gz = GZdopen(dup(fileno(in)),"r") ){ */ if( gz ){ LOGX_gunzip++; setCloseOnFork("GUNZIPstart",fd); em = gzerror(gz,&en); /* while( 0 < (rcc = gzread(gz,buf,sizeof(buf))) ){ */ for( gi = 0;; gi++ ){ if( gotsigTERM("gunzip gi=%d em=%X",gi,p2i(em)) ){ if( numthreads() ){ if( em ){ putfLog("thread-gunzip gi=%d _exit() em=(%s)",gi,em?em:""); _exit(0); } thread_exit(0); } break; } if( nonblock ){ if( 0 < gi ){ /* setNonblockingIO(fileno(in),0); */ setNonblockingIO(fd,0); nonblock = 0; } } /* if( 0 < size && inputReady(fileno(in),NULL) == 0 ){ */ /* if( 0 < size && inputReady(fd,NULL) == 0 ){ */ if( eof == 0 ) if( 0 < size ) if( ready = inputReady(fd,&rd) ){ if( ready == 2 ){ /* both PS_IN and PS_PRI */ eof = 1; } }else{ //fprintf(stderr,"[%d] -- gzread#%d %d / %d FLUSH\n",getpid(),gi,rcc,size); fflush(out); } ready = fPollIn(in,10*1000); errno = 0; rcc = gzread(gz,buf,QVSSize(buf,ibz)); serrno = errno; if( rcc <= 0 ){ break; } //fprintf(stderr,"[%d] -- gzread %d / %d\n",getpid(),rcc,size); wcc = fwrite(buf,1,rcc,out); /* this fflush seems significant */ werr = fflush(out); if( wcc < rcc || werr || ferror(out) || gotSIGPIPE() ){ porting_dbg("+++EPIPE[%d] gunzip fwrite() %d/%d err=%d/%d %d SIG*%d",fileno(out),wcc,rcc,werr,ferror(out),size,gotSIGPIPE()); break; } size += rcc; if( size < sizeof(buf) ){ fflush(out); }else{ ibz = sizeof(buf); } } fflush(out); if( rcc < 0 || size == 0 ){ em = gzerror(gz,&en); ef = gzeof(gz); if( en == -1 /* see errno */ && serrno == 0 ){ /* no error */ }else{ daemonlog("F","FATAL: gzread(%d)=%d/%d eof=%d %d %s %d\n", fd,rcc,size,ef,en,em,serrno); porting_dbg("FATAL: gzread(%d)=%d/%d eof=%d %d %s", fd,rcc,size,ef,en,em); if( lTHREAD() ) fprintf(stderr,"--[%d]gzread(%d)=%d/%d eof=%d %d %s\n", getpid(),fd,rcc,size,ef,en,em); } } clearCloseOnFork("GUNZIPend",fd); GZclose(gz); if( isWindowsCE() || lMULTIST() ){ /* duplicated close of fd is harmful */ }else if( isWindows() ) close(fd); fseek(out,0,0); syslog_DEBUG("(%f)gunzipFilter -> %d\n",Time()-Start,size); if( lTHREAD() ) if( 0 < inlen ) syslog_ERROR("###GUNZIP filter %d/%d\n",inlen,size); return size; } return 0; }
int gzipFilterX(FILE *in,FILE *out,SyncXF syncf,void *sp,int si){ gzFile gz; int len,rcc; CStr(buf,1024*8); int size; int gsize; int wcc; int bcc = 0; double Start = Time(); double Prevf = 0; int ibz = sizeof(buf); int gi; int fd = -1; int ofd = fileno(out); int xfd; int zerr = 0; /* int rready = -1; */ errno = 0; fd = dup(fileno(out)); if( fd < 0 ){ syslog_ERROR("--gzipFilter[%d]<-[%d] errno=%d\n",fd,ofd,errno); return -1; } /* if( 0 <= GZIPready ) rready = dup(GZIPready); */ len = 0; /* if( gz = GZdopen(dup(fileno(out)),"w") ){ */ if( file_isSOCKET(ofd) || file_ISSOCK(ofd) ) if( !IsConnected(ofd,NULL) || !IsAlive(ofd) ){ fprintf(stderr,"[%d.%X] gzip DISCONN\n",getpid(),getthreadid()); fprintf(stderr,"[%d.%X] gzip DISCONN fd[%d] con=%d isSOCK=%d,%d,%d\n", getpid(),getthreadid(),ofd,IsConnected(ofd,NULL), file_isSOCKET(ofd),file_ISSOCK(ofd),file_issock(ofd)); sendsync(rready,1); close(fd); return -1; } gz = GZdopen(fd,"w"); if( file_isSOCKET(ofd) || file_ISSOCK(ofd) ) if( !IsConnected(ofd,NULL) || !IsAlive(ofd) ){ fprintf(stderr,"[%d.%X] gzip DISCONN gx=%d\n",getpid(),getthreadid(),p2i(gz)); fprintf(stderr,"[%d.%X] gzip DISCONN fd[%d] con=%d isSOCK=%d,%d,%d\n", getpid(),getthreadid(),ofd,IsConnected(ofd,NULL), file_isSOCKET(ofd),file_ISSOCK(ofd),file_issock(ofd)); close(fd); sendsync(rready,2); close(fd); return -1; } if( gz ){ LOGX_gzip++; if( Gzip_NoFlush ){ GZDBG(stderr,"-- %X gzip flush disabled(%d)\n", TID,Gzip_NoFlush); } Prevf = Time(); sendsync(rready,0); setCloseOnFork("GZIPstart",fd); /* while( rcc = fread(buf,1,sizeof(buf),in) ){ */ for( gi = 0;; gi++ ){ if( gotsigTERM("gzip gi=%d",gi) ){ if( numthreads() && !ismainthread() ){ thread_exit(0); } break; } if( !Gzip_NoFlush ) if( bcc ) if( 0 < len && finputReady(in,NULL) == 0 ){ zerr = gzflush(gz,Z_SYNC_FLUSH); if( zerr ){ porting_dbg("+++EPIPE[%d] gzflush() zerr=%d %d SIG*%d",fd,zerr,len,gotSIGPIPE()); } bcc = 0; } if( lSINGLEP() ) /* could be generic */ { if( 0 < len ) if( !Gzip_NoFlush || 4 < gi && 5 < Time()-Prevf ){ GZDBG(stderr,"-- %X gzip flush %d(%f) %d/%d\n", TID,Gzip_NoFlush,Time()-Start,len,gi); Prevf = Time(); zerr = gzflush(gz,Z_SYNC_FLUSH); bcc = 0; if( zerr ){ GZDBG(stderr,"-- %X gzip gzflush()%d err=%d\n", TID,len,zerr); break; } } } /* rcc = fread(buf,1,sizeof(buf),in); */ rcc = xread(in,AVStr(buf),QVSSize(buf,ibz)); if( rcc <= 0 ){ break; } wcc = gzwrite(gz,buf,rcc); //fprintf(stderr,"[%d] Gzwrite %d/%d / %d\n",getpid(),wcc,rcc,len); if( wcc <= 0 ){ porting_dbg("+++EPIPE[%d] gzwrite() %d/%d %d SIG*%d",fd,wcc,rcc,len,gotSIGPIPE()); fprintf(stderr,"[%d] Gzwrite %d/%d / %d\n",getpid(),wcc,rcc,len); break; } if( wcc != rcc ){ syslog_ERROR("gzwrite %d/%d\n",wcc,rcc); } if( 0 < wcc ){ bcc += wcc; } if( sizeof(buf) <= len ){ ibz = sizeof(buf); } if( !Gzip_NoFlush ) if( bcc ) if( sizeof(buf) <= bcc || len < 16*1024 ){ zerr = gzflush(gz,Z_SYNC_FLUSH); bcc = 0; } if( zerr || gotSIGPIPE() ){ porting_dbg("+++EPIPE[%d] gzflush() zerr=%d %d SIG*%d",fd,zerr,len,gotSIGPIPE()); break; } len += rcc; } if( len == 0 ){ const char *em; int en; int ef; em = gzerror(gz,&en); ef = gzeof(gz); if( en == -1 /* see errno */ && errno == 0 ){ /* no error */ }else{ daemonlog("F","FATAL: gzwrite(%d)=%d/%d eof=%d %d %s\n", fd,len,bcc,ef,en,em); porting_dbg("FATAL: gzwrite(%d)=%d/%d eof=%d %d %s", fd,len,bcc,ef,en,em); } } clearCloseOnFork("GZIPend",fd); gzflush(gz,Z_SYNC_FLUSH); xfd = dup(fd); gsize = GZtell(gz); GZclose(gz); if( isWindowsCE() || lMULTIST() ){ /* duplicated close of fd is harmful */ }else if( isWindows() ) close(fd); /* to clear osf-handle mapping */ Lseek(xfd,0,2); size = Lseek(xfd,0,1); Lseek(xfd,0,0); close(xfd); syslog_DEBUG("(%f)gzipFilter %d -> %d / %d\n",Time()-Start, len,gsize,size); return len; } sendsync(rready,3); close(fd); return 0; }