/* ** Compress the concatenation of a blobs pIn1 and pIn2. Store the result ** in pOut. ** ** pOut must be either uninitialized or must be the same as either pIn1 or ** pIn2. */ void blob_compress2(Blob *pIn1, Blob *pIn2, Blob *pOut){ unsigned int nIn = blob_size(pIn1) + blob_size(pIn2); unsigned int nOut = 13 + nIn + (nIn+999)/1000; unsigned char *outBuf; z_stream stream; Blob temp; blob_zero(&temp); blob_resize(&temp, nOut+4); outBuf = (unsigned char*)blob_buffer(&temp); outBuf[0] = nIn>>24 & 0xff; outBuf[1] = nIn>>16 & 0xff; outBuf[2] = nIn>>8 & 0xff; outBuf[3] = nIn & 0xff; stream.zalloc = (alloc_func)0; stream.zfree = (free_func)0; stream.opaque = 0; stream.avail_out = nOut; stream.next_out = &outBuf[4]; deflateInit(&stream, 9); stream.avail_in = blob_size(pIn1); stream.next_in = (unsigned char*)blob_buffer(pIn1); deflate(&stream, 0); stream.avail_in = blob_size(pIn2); stream.next_in = (unsigned char*)blob_buffer(pIn2); deflate(&stream, 0); deflate(&stream, Z_FINISH); blob_resize(&temp, stream.total_out + 4); deflateEnd(&stream); if( pOut==pIn1 ) blob_reset(pOut); if( pOut==pIn2 ) blob_reset(pOut); assert_blob_is_reset(pOut); *pOut = temp; }
/* ** Uncompress blob pIn and store the result in pOut. It is ok for pIn and ** pOut to be the same blob. ** ** pOut must be either uninitialized or the same as pIn. */ int blob_uncompress(Blob *pIn, Blob *pOut){ unsigned int nOut; unsigned char *inBuf; unsigned int nIn = blob_size(pIn); Blob temp; int rc; unsigned long int nOut2; if( nIn<=4 ){ return 0; } inBuf = (unsigned char*)blob_buffer(pIn); nOut = (inBuf[0]<<24) + (inBuf[1]<<16) + (inBuf[2]<<8) + inBuf[3]; blob_zero(&temp); blob_resize(&temp, nOut+1); nOut2 = (long int)nOut; rc = uncompress((unsigned char*)blob_buffer(&temp), &nOut2, &inBuf[4], nIn - 4); if( rc!=Z_OK ){ blob_reset(&temp); return 1; } blob_resize(&temp, nOut2); if( pOut==pIn ) blob_reset(pOut); assert_blob_is_reset(pOut); *pOut = temp; return 0; }
/* ** Initialize a blob to be the content of a file. If the filename ** is blank or "-" then read from standard input. ** ** Any prior content of the blob is discarded, not freed. ** ** Return the number of bytes read. Calls fossil_fatal() error (i.e. ** it exit()s and does not return). */ int blob_read_from_file(Blob *pBlob, const char *zFilename){ int size, got; FILE *in; if( zFilename==0 || zFilename[0]==0 || (zFilename[0]=='-' && zFilename[1]==0) ){ return blob_read_from_channel(pBlob, stdin, -1); } size = file_wd_size(zFilename); blob_zero(pBlob); if( size<0 ){ fossil_fatal("no such file: %s", zFilename); } if( size==0 ){ return 0; } blob_resize(pBlob, size); in = fossil_fopen(zFilename, "rb"); if( in==0 ){ fossil_fatal("cannot open %s for reading", zFilename); } got = fread(blob_buffer(pBlob), 1, size, in); fclose(in); if( got<size ){ blob_resize(pBlob, got); } return got; }
/* ** Initialize a blob to the data on an input channel. Return ** the number of bytes read into the blob. Any prior content ** of the blob is discarded, not freed. */ int blob_read_from_channel(Blob *pBlob, FILE *in, int nToRead){ size_t n; blob_zero(pBlob); if( nToRead<0 ){ char zBuf[10000]; while( !feof(in) ){ n = fread(zBuf, 1, sizeof(zBuf), in); if( n>0 ){ blob_append(pBlob, zBuf, n); } } }else{ blob_resize(pBlob, nToRead); n = fread(blob_buffer(pBlob), 1, nToRead, in); blob_resize(pBlob, n); } return blob_size(pBlob); }
/* ** Compress a blob pIn. Store the result in pOut. It is ok for pIn and ** pOut to be the same blob. ** ** pOut must either be the same as pIn or else uninitialized. */ void blob_compress(Blob *pIn, Blob *pOut){ unsigned int nIn = blob_size(pIn); unsigned int nOut = 13 + nIn + (nIn+999)/1000; unsigned long int nOut2; unsigned char *outBuf; Blob temp; blob_zero(&temp); blob_resize(&temp, nOut+4); outBuf = (unsigned char*)blob_buffer(&temp); outBuf[0] = nIn>>24 & 0xff; outBuf[1] = nIn>>16 & 0xff; outBuf[2] = nIn>>8 & 0xff; outBuf[3] = nIn & 0xff; nOut2 = (long int)nOut; compress(&outBuf[4], &nOut2, (unsigned char*)blob_buffer(pIn), blob_size(pIn)); if( pOut==pIn ) blob_reset(pOut); assert_blob_is_reset(pOut); *pOut = temp; blob_resize(pOut, nOut2+4); }
/* ** Convert every \n character in the given blob into \r\n. */ void blob_add_cr(Blob *p){ char *z = p->aData; int j = p->nUsed; int i, n; for(i=n=0; i<j; i++){ if( z[i]=='\n' ) n++; } j += n; if( j>=p->nAlloc ){ blob_resize(p, j); z = p->aData; } p->nUsed = j; z[j] = 0; while( j>i ){ if( (z[--j] = z[--i]) =='\n' ){ z[--j] = '\r'; } } }
/* ** Check to see if the directory named in zPath is the top of a checkout. ** In other words, check to see if directory pPath contains a file named ** "_vcs_" or ".fslckout". Return true or false. */ int vfile_top_of_checkout(const char *zPath){ char *zFile; int fileFound = 0; zFile = mprintf("%s/_vcs_", zPath); fileFound = file_size(zFile)>=1024; vcs_free(zFile); if( !fileFound ){ zFile = mprintf("%s/.fslckout", zPath); fileFound = file_size(zFile)>=1024; vcs_free(zFile); } /* ** Load into table SFILE the name of every ordinary file in ** the directory pPath. Omit the first nPrefix characters of ** of pPath when inserting into the SFILE table. ** ** Subdirectories are scanned recursively. ** Omit files named in VFILE. ** ** Files whose names begin with "." are omitted unless allFlag is true. ** ** Any files or directories that match the glob pattern pIgnore are ** excluded from the scan. Name matching occurs after the first ** nPrefix characters are elided from the filename. */ void vfile_scan(Blob *pPath, int nPrefix, int allFlag, Glob *pIgnore){ DIR *d; int origSize; const char *zDir; struct dirent *pEntry; int skipAll = 0; static Stmt ins; static int depth = 0; char *zMbcs; origSize = blob_size(pPath); if( pIgnore ){ blob_appendf(pPath, "/"); if( glob_match(pIgnore, &blob_str(pPath)[nPrefix+1]) ) skipAll = 1; blob_resize(pPath, origSize); } if( skipAll ) return; if( depth==0 ){ db_prepare(&ins, "INSERT OR IGNORE INTO sfile(x) SELECT :file" " WHERE NOT EXISTS(SELECT 1 FROM vfile WHERE pathname=:file)" ); } depth++; zDir = blob_str(pPath); zMbcs = vcs_utf8_to_mbcs(zDir); d = opendir(zMbcs); if( d ){ while( (pEntry=readdir(d))!=0 ){ char *zPath; char *zUtf8; if( pEntry->d_name[0]=='.' ){ if( !allFlag ) continue; if( pEntry->d_name[1]==0 ) continue; if( pEntry->d_name[1]=='.' && pEntry->d_name[2]==0 ) continue; } zUtf8 = vcs_mbcs_to_utf8(pEntry->d_name); blob_appendf(pPath, "/%s", zUtf8); vcs_mbcs_free(zUtf8); zPath = blob_str(pPath); if( glob_match(pIgnore, &zPath[nPrefix+1]) ){ /* do nothing */ }else if( file_wd_isdir(zPath)==1 ){ if( !vfile_top_of_checkout(zPath) ){ vfile_scan(pPath, nPrefix, allFlag, pIgnore); } }else if( file_wd_isfile_or_link(zPath) ){ db_bind_text(&ins, ":file", &zPath[nPrefix+1]); db_step(&ins); db_reset(&ins); } blob_resize(pPath, origSize); } closedir(d); } vcs_mbcs_free(zMbcs); depth--; if( depth==0 ){ db_finalize(&ins); } }
/* ** Make sure a blob is nul-terminated and is not a pointer to unmanaged ** space. Return a pointer to the data. */ char *blob_materialize(Blob *pBlob){ blob_resize(pBlob, pBlob->nUsed); return pBlob->aData; }
/* ** Given the RID for a checkin, construct a tarball containing ** all files in that checkin ** ** If RID is for an object that is not a real manifest, then the ** resulting tarball contains a single file which is the RID ** object. ** ** If the RID object does not exist in the repository, then ** pTar is zeroed. ** ** zDir is a "synthetic" subdirectory which all files get ** added to as part of the tarball. It may be 0 or an empty string, in ** which case it is ignored. The intention is to create a tarball which ** politely expands into a subdir instead of filling your current dir ** with source files. For example, pass a UUID or "ProjectName". ** */ void tarball_of_checkin(int rid, Blob *pTar, const char *zDir){ Blob mfile, hash, file; Manifest *pManifest; ManifestFile *pFile; Blob filename; int nPrefix; char *zName; unsigned int mTime; content_get(rid, &mfile); if( blob_size(&mfile)==0 ){ blob_zero(pTar); return; } blob_zero(&hash); blob_zero(&filename); if( zDir && zDir[0] ){ blob_appendf(&filename, "%s/", zDir); } nPrefix = blob_size(&filename); pManifest = manifest_get(rid, CFTYPE_MANIFEST); if( pManifest ){ mTime = (pManifest->rDate - 2440587.5)*86400.0; tar_begin(mTime); if( db_get_boolean("manifest", 0) ){ blob_append(&filename, "manifest", -1); zName = blob_str(&filename); tar_add_file(zName, &mfile, 0, mTime); sha1sum_blob(&mfile, &hash); blob_reset(&mfile); blob_append(&hash, "\n", 1); blob_resize(&filename, nPrefix); blob_append(&filename, "manifest.uuid", -1); zName = blob_str(&filename); tar_add_file(zName, &hash, 0, mTime); blob_reset(&hash); } manifest_file_rewind(pManifest); while( (pFile = manifest_file_next(pManifest,0))!=0 ){ int fid = uuid_to_rid(pFile->zUuid, 0); if( fid ){ content_get(fid, &file); blob_resize(&filename, nPrefix); blob_append(&filename, pFile->zName, -1); zName = blob_str(&filename); tar_add_file(zName, &file, manifest_file_mperm(pFile), mTime); blob_reset(&file); } } }else{ sha1sum_blob(&mfile, &hash); blob_append(&filename, blob_str(&hash), 16); zName = blob_str(&filename); mTime = db_int64(0, "SELECT (julianday('now') - 2440587.5)*86400.0;"); tar_begin(mTime); tar_add_file(zName, &mfile, 0, mTime); } manifest_destroy(pManifest); blob_reset(&mfile); blob_reset(&filename); tar_finish(pTar); }
/* ** Scans the specified base directory for any directories within it, while ** keeping a count of how many files they each contains, either directly or ** indirectly. ** ** Subdirectories are scanned recursively. ** Omit files named in VFILE. ** ** Directories whose names begin with "." are omitted unless the SCAN_ALL ** flag is set. ** ** Any directories that match the glob patterns pIgnore* are excluded from ** the scan. Name matching occurs after the first nPrefix characters are ** elided from the filename. ** ** Returns the total number of files found. */ int vfile_dir_scan( Blob *pPath, /* Base directory to be scanned */ int nPrefix, /* Number of bytes in base directory name */ unsigned scanFlags, /* Zero or more SCAN_xxx flags */ Glob *pIgnore1, /* Do not add directories that match this GLOB */ Glob *pIgnore2, /* Omit directories matching this GLOB too */ Glob *pIgnore3 /* Omit directories matching this GLOB too */ ){ int result = 0; DIR *d; int origSize; struct dirent *pEntry; int skipAll = 0; static Stmt ins; static Stmt upd; static int depth = 0; void *zNative; origSize = blob_size(pPath); if( pIgnore1 || pIgnore2 || pIgnore3 ){ blob_appendf(pPath, "/"); if( glob_match(pIgnore1, &blob_str(pPath)[nPrefix+1]) ) skipAll = 1; if( glob_match(pIgnore2, &blob_str(pPath)[nPrefix+1]) ) skipAll = 1; if( glob_match(pIgnore3, &blob_str(pPath)[nPrefix+1]) ) skipAll = 1; blob_resize(pPath, origSize); } if( skipAll ) return result; if( depth==0 ){ db_multi_exec("DROP TABLE IF EXISTS dscan_temp;" "CREATE TEMP TABLE dscan_temp(" " x TEXT PRIMARY KEY %s, y INTEGER)", filename_collation()); db_prepare(&ins, "INSERT OR IGNORE INTO dscan_temp(x, y) SELECT :file, :count" " WHERE NOT EXISTS(SELECT 1 FROM vfile WHERE" " pathname GLOB :file || '/*' %s)", filename_collation() ); db_prepare(&upd, "UPDATE OR IGNORE dscan_temp SET y = coalesce(y, 0) + 1" " WHERE x=:file %s", filename_collation() ); } depth++; zNative = fossil_utf8_to_filename(blob_str(pPath)); d = opendir(zNative); if( d ){ while( (pEntry=readdir(d))!=0 ){ char *zOrigPath; char *zPath; char *zUtf8; if( pEntry->d_name[0]=='.' ){ if( (scanFlags & SCAN_ALL)==0 ) continue; if( pEntry->d_name[1]==0 ) continue; if( pEntry->d_name[1]=='.' && pEntry->d_name[2]==0 ) continue; } zOrigPath = mprintf("%s", blob_str(pPath)); zUtf8 = fossil_filename_to_utf8(pEntry->d_name); blob_appendf(pPath, "/%s", zUtf8); zPath = blob_str(pPath); if( glob_match(pIgnore1, &zPath[nPrefix+1]) || glob_match(pIgnore2, &zPath[nPrefix+1]) || glob_match(pIgnore3, &zPath[nPrefix+1]) ){ /* do nothing */ }else if( file_wd_isdir(zPath)==1 ){ if( (scanFlags & SCAN_NESTED) || !vfile_top_of_checkout(zPath) ){ char *zSavePath = mprintf("%s", zPath); int count = vfile_dir_scan(pPath, nPrefix, scanFlags, pIgnore1, pIgnore2, pIgnore3); db_bind_text(&ins, ":file", &zSavePath[nPrefix+1]); db_bind_int(&ins, ":count", count); db_step(&ins); db_reset(&ins); fossil_free(zSavePath); result += count; /* found X normal files? */ } }else if( file_wd_isfile_or_link(zPath) ){ db_bind_text(&upd, ":file", zOrigPath); db_step(&upd); db_reset(&upd); result++; /* found 1 normal file */ } fossil_filename_free(zUtf8); blob_resize(pPath, origSize); fossil_free(zOrigPath); } closedir(d); } fossil_filename_free(zNative); depth--; if( depth==0 ){ db_finalize(&upd); db_finalize(&ins); } return result; }
/* ** Load into table SFILE the name of every ordinary file in ** the directory pPath. Omit the first nPrefix characters of ** of pPath when inserting into the SFILE table. ** ** Subdirectories are scanned recursively. ** Omit files named in VFILE. ** ** Files whose names begin with "." are omitted unless the SCAN_ALL ** flag is set. ** ** Any files or directories that match the glob patterns pIgnore* ** are excluded from the scan. Name matching occurs after the ** first nPrefix characters are elided from the filename. */ void vfile_scan( Blob *pPath, /* Directory to be scanned */ int nPrefix, /* Number of bytes in directory name */ unsigned scanFlags, /* Zero or more SCAN_xxx flags */ Glob *pIgnore1, /* Do not add files that match this GLOB */ Glob *pIgnore2 /* Omit files matching this GLOB too */ ){ DIR *d; int origSize; struct dirent *pEntry; int skipAll = 0; static Stmt ins; static int depth = 0; void *zNative; origSize = blob_size(pPath); if( pIgnore1 || pIgnore2 ){ blob_appendf(pPath, "/"); if( glob_match(pIgnore1, &blob_str(pPath)[nPrefix+1]) ) skipAll = 1; if( glob_match(pIgnore2, &blob_str(pPath)[nPrefix+1]) ) skipAll = 1; blob_resize(pPath, origSize); } if( skipAll ) return; if( depth==0 ){ db_prepare(&ins, "INSERT OR IGNORE INTO sfile(x) SELECT :file" " WHERE NOT EXISTS(SELECT 1 FROM vfile WHERE" " pathname=:file %s)", filename_collation() ); } depth++; zNative = fossil_utf8_to_filename(blob_str(pPath)); d = opendir(zNative); if( d ){ while( (pEntry=readdir(d))!=0 ){ char *zPath; char *zUtf8; if( pEntry->d_name[0]=='.' ){ if( (scanFlags & SCAN_ALL)==0 ) continue; if( pEntry->d_name[1]==0 ) continue; if( pEntry->d_name[1]=='.' && pEntry->d_name[2]==0 ) continue; } zUtf8 = fossil_filename_to_utf8(pEntry->d_name); blob_appendf(pPath, "/%s", zUtf8); zPath = blob_str(pPath); if( glob_match(pIgnore1, &zPath[nPrefix+1]) || glob_match(pIgnore2, &zPath[nPrefix+1]) ){ /* do nothing */ }else if( file_wd_isdir(zPath)==1 ){ if( !vfile_top_of_checkout(zPath) ){ vfile_scan(pPath, nPrefix, scanFlags, pIgnore1, pIgnore2); } }else if( file_wd_isfile_or_link(zPath) ){ if( (scanFlags & SCAN_TEMP)==0 || is_temporary_file(zUtf8) ){ db_bind_text(&ins, ":file", &zPath[nPrefix+1]); db_step(&ins); db_reset(&ins); } } fossil_filename_free(zUtf8); blob_resize(pPath, origSize); } closedir(d); } fossil_filename_free(zNative); depth--; if( depth==0 ){ db_finalize(&ins); } }