/*
** COMMAND: test-compress
*/
void compress_cmd(void){
Blob f;
if( g.argc!=4 ) usage("INPUTFILE OUTPUTFILE");
blob_read_from_file(&f, g.argv[2]);
blob_compress(&f, &f);
blob_write_to_file(&f, g.argv[3]);
}
/*
** Implement the "fossil bundle append BUNDLE FILE..." command. Add
** the named files into the BUNDLE. Create the BUNDLE if it does not
** alraedy exist.
*/
static void bundle_append_cmd(void){
Blob content, hash;
int i;
Stmt q;
verify_all_options();
bundle_attach_file(g.argv[3], "b1", 1);
db_prepare(&q,
"INSERT INTO bblob(blobid, uuid, sz, delta, data, notes) "
"VALUES(NULL, $uuid, $sz, NULL, $data, $filename)");
db_begin_transaction();
for(i=4; i<g.argc; i++){
int sz;
blob_read_from_file(&content, g.argv[i]);
sz = blob_size(&content);
sha1sum_blob(&content, &hash);
blob_compress(&content, &content);
db_bind_text(&q, "$uuid", blob_str(&hash));
db_bind_int(&q, "$sz", sz);
db_bind_blob(&q, "$data", &content);
db_bind_text(&q, "$filename", g.argv[i]);
db_step(&q);
db_reset(&q);
blob_reset(&content);
blob_reset(&hash);
}
db_end_transaction(0);
db_finalize(&q);
}
/*
** Change the storage of rid so that it is a delta of srcid.
**
** If rid is already a delta from some other place then no
** conversion occurs and this is a no-op unless force==1.
**
** Never generate a delta that carries a private artifact into a public
** artifact. Otherwise, when we go to send the public artifact on a
** sync operation, the other end of the sync will never be able to receive
** the source of the delta. It is OK to delta private->private and
** public->private and public->public. Just no private->public delta.
**
** If srcid is a delta that depends on rid, then srcid is
** converted to undeltaed text.
**
** If either rid or srcid contain less than 50 bytes, or if the
** resulting delta does not achieve a compression of at least 25%
** the rid is left untouched.
**
** Return 1 if a delta is made and 0 if no delta occurs.
*/
int content_deltify(int rid, int srcid, int force){
int s;
Blob data, src, delta;
Stmt s1, s2;
int rc = 0;
if( srcid==rid ) return 0;
if( !force && findSrcid(rid)>0 ) return 0;
if( content_is_private(srcid) && !content_is_private(rid) ){
return 0;
}
s = srcid;
while( (s = findSrcid(s))>0 ){
if( s==rid ){
content_undelta(srcid);
break;
}
}
content_get(srcid, &src);
if( blob_size(&src)<50 ){
blob_reset(&src);
return 0;
}
content_get(rid, &data);
if( blob_size(&data)<50 ){
blob_reset(&src);
blob_reset(&data);
return 0;
}
blob_delta_create(&src, &data, &delta);
if( blob_size(&delta) <= blob_size(&data)*0.75 ){
blob_compress(&delta, &delta);
db_prepare(&s1, "UPDATE blob SET content=:data WHERE rid=%d", rid);
db_prepare(&s2, "REPLACE INTO delta(rid,srcid)VALUES(%d,%d)", rid, srcid);
db_bind_blob(&s1, ":data", &delta);
db_begin_transaction();
db_exec(&s1);
db_exec(&s2);
db_end_transaction(0);
db_finalize(&s1);
db_finalize(&s2);
verify_before_commit(rid);
rc = 1;
}
blob_reset(&src);
blob_reset(&data);
blob_reset(&delta);
return rc;
}
/*
** Make sure the content at rid is the original content and is not a
** delta.
*/
void content_undelta(int rid){
if( findSrcid(rid)>0 ){
Blob x;
if( content_get(rid, &x) ){
Stmt s;
db_prepare(&s, "UPDATE blob SET content=:c, size=%d WHERE rid=%d",
blob_size(&x), rid);
blob_compress(&x, &x);
db_bind_blob(&s, ":c", &x);
db_exec(&s);
db_finalize(&s);
blob_reset(&x);
db_multi_exec("DELETE FROM delta WHERE rid=%d", rid);
}
}
}
/*
** COMMAND: test-cycle-compress
**
** Compress and uncompress each file named on the command line.
** Verify that the original content is recovered.
*/
void test_cycle_compress(void){
int i;
Blob b1, b2, b3;
for(i=2; i<g.argc; i++){
blob_read_from_file(&b1, g.argv[i]);
blob_compress(&b1, &b2);
blob_uncompress(&b2, &b3);
if( blob_compare(&b1, &b3) ){
fossil_fatal("compress/uncompress cycle failed for %s", g.argv[i]);
}
blob_reset(&b1);
blob_reset(&b2);
blob_reset(&b3);
}
fossil_print("ok\n");
}
/*
** Insert an artifact into the BLOB table if it isn't there already.
** If zMark is not zero, create a cross-reference from that mark back
** to the newly inserted artifact.
**
** If saveUuid is true, then pContent is a commit record. Record its
** UUID in gg.zPrevCheckin.
*/
static int fast_insert_content(Blob *pContent, const char *zMark, int saveUuid){
Blob hash;
Blob cmpr;
int rid;
sha1sum_blob(pContent, &hash);
rid = db_int(0, "SELECT rid FROM blob WHERE uuid=%B", &hash);
if( rid==0 ){
static Stmt ins;
db_static_prepare(&ins,
"INSERT INTO blob(uuid, size, content) VALUES(:uuid, :size, :content)"
);
db_bind_text(&ins, ":uuid", blob_str(&hash));
db_bind_int(&ins, ":size", gg.nData);
blob_compress(pContent, &cmpr);
db_bind_blob(&ins, ":content", &cmpr);
db_step(&ins);
db_reset(&ins);
blob_reset(&cmpr);
rid = db_last_insert_rowid();
}
if( zMark ){
db_multi_exec(
"INSERT OR IGNORE INTO xmark(tname, trid, tuuid)"
"VALUES(%Q,%d,%B)",
zMark, rid, &hash
);
db_multi_exec(
"INSERT OR IGNORE INTO xmark(tname, trid, tuuid)"
"VALUES(%B,%d,%B)",
&hash, rid, &hash
);
}
if( saveUuid ){
fossil_free(gg.zPrevCheckin);
gg.zPrevCheckin = fossil_strdup(blob_str(&hash));
}
blob_reset(&hash);
return rid;
}
/*
** Write content into the database. Return the record ID. If the
** content is already in the database, just return the record ID.
**
** If srcId is specified, then pBlob is delta content from
** the srcId record. srcId might be a phantom.
**
** pBlob is normally uncompressed text. But if nBlob>0 then the
** pBlob value has already been compressed and nBlob is its uncompressed
** size. If nBlob>0 then zUuid must be valid.
**
** zUuid is the UUID of the artifact, if it is specified. When srcId is
** specified then zUuid must always be specified. If srcId is zero,
** and zUuid is zero then the correct zUuid is computed from pBlob.
**
** If the record already exists but is a phantom, the pBlob content
** is inserted and the phatom becomes a real record.
**
** The original content of pBlob is not disturbed. The caller continues
** to be responsible for pBlob. This routine does *not* take over
** responsibility for freeing pBlob.
*/
int content_put_ex(
Blob *pBlob, /* Content to add to the repository */
const char *zUuid, /* SHA1 hash of reconstructed pBlob */
int srcId, /* pBlob is a delta from this entry */
int nBlob, /* pBlob is compressed. Original size is this */
int isPrivate /* The content should be marked private */
){
int size;
int rid;
Stmt s1;
Blob cmpr;
Blob hash;
int markAsUnclustered = 0;
int isDephantomize = 0;
assert( g.repositoryOpen );
assert( pBlob!=0 );
assert( srcId==0 || zUuid!=0 );
if( zUuid==0 ){
assert( nBlob==0 );
sha1sum_blob(pBlob, &hash);
}else{
blob_init(&hash, zUuid, -1);
}
if( nBlob ){
size = nBlob;
}else{
size = blob_size(pBlob);
if( srcId ){
size = delta_output_size(blob_buffer(pBlob), size);
}
}
db_begin_transaction();
/* Check to see if the entry already exists and if it does whether
** or not the entry is a phantom
*/
db_prepare(&s1, "SELECT rid, size FROM blob WHERE uuid=%B", &hash);
if( db_step(&s1)==SQLITE_ROW ){
rid = db_column_int(&s1, 0);
if( db_column_int(&s1, 1)>=0 || pBlob==0 ){
/* Either the entry is not a phantom or it is a phantom but we
** have no data with which to dephantomize it. In either case,
** there is nothing for us to do other than return the RID. */
db_finalize(&s1);
db_end_transaction(0);
return rid;
}
}else{
rid = 0; /* No entry with the same UUID currently exists */
markAsUnclustered = 1;
}
db_finalize(&s1);
/* Construct a received-from ID if we do not already have one */
if( g.rcvid==0 ){
db_multi_exec(
"INSERT INTO rcvfrom(uid, mtime, nonce, ipaddr)"
"VALUES(%d, julianday('now'), %Q, %Q)",
g.userUid, g.zNonce, g.zIpAddr
);
g.rcvid = db_last_insert_rowid();
}
if( nBlob ){
cmpr = pBlob[0];
}else{
blob_compress(pBlob, &cmpr);
}
if( rid>0 ){
/* We are just adding data to a phantom */
db_prepare(&s1,
"UPDATE blob SET rcvid=%d, size=%d, content=:data WHERE rid=%d",
g.rcvid, size, rid
);
db_bind_blob(&s1, ":data", &cmpr);
db_exec(&s1);
db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
if( srcId==0 || content_is_available(srcId) ){
isDephantomize = 1;
content_mark_available(rid);
}
}else{
/* We are creating a new entry */
db_prepare(&s1,
"INSERT INTO blob(rcvid,size,uuid,content)"
"VALUES(%d,%d,'%b',:data)",
g.rcvid, size, &hash
);
db_bind_blob(&s1, ":data", &cmpr);
db_exec(&s1);
rid = db_last_insert_rowid();
if( !pBlob ){
db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
}
if( g.markPrivate || isPrivate ){
db_multi_exec("INSERT INTO private VALUES(%d)", rid);
markAsUnclustered = 0;
}
}
if( nBlob==0 ) blob_reset(&cmpr);
/* If the srcId is specified, then the data we just added is
** really a delta. Record this fact in the delta table.
*/
if( srcId ){
db_multi_exec("REPLACE INTO delta(rid,srcid) VALUES(%d,%d)", rid, srcId);
}
if( !isDephantomize && bag_find(&contentCache.missing, rid) &&
(srcId==0 || content_is_available(srcId)) ){
content_mark_available(rid);
}
if( isDephantomize ){
after_dephantomize(rid, 0);
}
/* Add the element to the unclustered table if has never been
** previously seen.
*/
if( markAsUnclustered ){
db_multi_exec("INSERT OR IGNORE INTO unclustered VALUES(%d)", rid);
}
/* Finish the transaction and cleanup */
db_finalize(&s1);
db_end_transaction(0);
blob_reset(&hash);
/* Make arrangements to verify that the data can be recovered
** before we commit */
verify_before_commit(rid);
return rid;
}
/* fossil bundle export BUNDLE ?OPTIONS?
**
** OPTIONS:
** --branch BRANCH --from TAG --to TAG
** --checkin TAG
** --standalone
*/
static void bundle_export_cmd(void){
int bStandalone = find_option("standalone",0,0)!=0;
int mnToBundle; /* Minimum RID in the bundle */
Stmt q;
/* Decode the arguments (like --branch) that specify which artifacts
** should be in the bundle */
db_multi_exec("CREATE TEMP TABLE tobundle(rid INTEGER PRIMARY KEY);");
subtree_from_arguments("tobundle");
find_checkin_associates("tobundle", 0);
verify_all_options();
describe_artifacts("IN tobundle");
if( g.argc!=4 ) usage("export BUNDLE ?OPTIONS?");
/* Create the new bundle */
bundle_attach_file(g.argv[3], "b1", 1);
db_begin_transaction();
/* Add 'mtime' and 'project-code' entries to the bconfig table */
db_multi_exec(
"INSERT INTO bconfig(bcname,bcvalue)"
" VALUES('mtime',datetime('now'));"
);
db_multi_exec(
"INSERT INTO bconfig(bcname,bcvalue)"
" SELECT name, value FROM config"
" WHERE name IN ('project-code');"
);
/* Directly copy content from the repository into the bundle as long
** as the repository content is a delta from some other artifact that
** is also in the bundle.
*/
db_multi_exec(
"REPLACE INTO bblob(blobid,uuid,sz,delta,data,notes) "
" SELECT"
" tobundle.rid,"
" blob.uuid,"
" blob.size,"
" delta.srcid,"
" blob.content,"
" (SELECT summary FROM description WHERE rid=blob.rid)"
" FROM tobundle, blob, delta"
" WHERE blob.rid=tobundle.rid"
" AND delta.rid=tobundle.rid"
" AND delta.srcid IN tobundle;"
);
/* For all the remaining artifacts, we need to construct their deltas
** manually.
*/
mnToBundle = db_int(0,"SELECT min(rid) FROM tobundle");
db_prepare(&q,
"SELECT rid FROM tobundle"
" WHERE rid NOT IN (SELECT blobid FROM bblob)"
" ORDER BY +rid;"
);
while( db_step(&q)==SQLITE_ROW ){
Blob content;
int rid = db_column_int(&q,0);
int deltaFrom = 0;
/* Get the raw, uncompressed content of the artifact into content */
content_get(rid, &content);
/* Try to find another artifact, not within the bundle, that is a
** plausible candidate for being a delta basis for the content. Set
** deltaFrom to the RID of that other artifact. Leave deltaFrom set
** to zero if the content should not be delta-compressed
*/
if( !bStandalone ){
if( db_exists("SELECT 1 FROM plink WHERE cid=%d",rid) ){
deltaFrom = db_int(0,
"SELECT max(cid) FROM plink"
" WHERE cid<%d", mnToBundle);
}else{
deltaFrom = db_int(0,
"SELECT max(fid) FROM mlink"
" WHERE fnid=(SELECT fnid FROM mlink WHERE fid=%d)"
" AND fid<%d", rid, mnToBundle);
}
}
/* Try to insert the insert the artifact as a delta
*/
if( deltaFrom ){
Blob basis, delta;
content_get(deltaFrom, &basis);
blob_delta_create(&basis, &content, &delta);
if( blob_size(&delta)>0.9*blob_size(&content) ){
deltaFrom = 0;
}else{
Stmt ins;
blob_compress(&delta, &delta);
db_prepare(&ins,
"REPLACE INTO bblob(blobid,uuid,sz,delta,data,notes)"
" SELECT %d, uuid, size, (SELECT uuid FROM blob WHERE rid=%d),"
" :delta, (SELECT summary FROM description WHERE rid=blob.rid)"
" FROM blob WHERE rid=%d", rid, deltaFrom, rid);
db_bind_blob(&ins, ":delta", &delta);
db_step(&ins);
db_finalize(&ins);
}
blob_reset(&basis);
blob_reset(&delta);
}
/* If unable to insert the artifact as a delta, insert full-text */
if( deltaFrom==0 ){
Stmt ins;
blob_compress(&content, &content);
db_prepare(&ins,
"REPLACE INTO bblob(blobid,uuid,sz,delta,data,notes)"
" SELECT rid, uuid, size, NULL, :content,"
" (SELECT summary FROM description WHERE rid=blob.rid)"
" FROM blob WHERE rid=%d", rid);
db_bind_blob(&ins, ":content", &content);
db_step(&ins);
db_finalize(&ins);
}
blob_reset(&content);
}
db_finalize(&q);
db_end_transaction(0);
}