/*
** Implement the "fossil bundle append BUNDLE FILE..." command. Add
** the named files into the BUNDLE. Create the BUNDLE if it does not
** alraedy exist.
*/
static void bundle_append_cmd(void){
Blob content, hash;
int i;
Stmt q;
verify_all_options();
bundle_attach_file(g.argv[3], "b1", 1);
db_prepare(&q,
"INSERT INTO bblob(blobid, uuid, sz, delta, data, notes) "
"VALUES(NULL, $uuid, $sz, NULL, $data, $filename)");
db_begin_transaction();
for(i=4; i<g.argc; i++){
int sz;
blob_read_from_file(&content, g.argv[i]);
sz = blob_size(&content);
sha1sum_blob(&content, &hash);
blob_compress(&content, &content);
db_bind_text(&q, "$uuid", blob_str(&hash));
db_bind_int(&q, "$sz", sz);
db_bind_blob(&q, "$data", &content);
db_bind_text(&q, "$filename", g.argv[i]);
db_step(&q);
db_reset(&q);
blob_reset(&content);
blob_reset(&hash);
}
db_end_transaction(0);
db_finalize(&q);
}
/*
** Extract an item from content from the bundle
*/
static void bundle_extract_item(
int blobid, /* ID of the item to extract */
Blob *pOut /* Write the content into this blob */
){
Stmt q;
Blob x, basis, h1, h2;
static Bag busy;
db_prepare(&q, "SELECT uuid, delta, data FROM bblob"
" WHERE blobid=%d", blobid);
if( db_step(&q)!=SQLITE_ROW ){
db_finalize(&q);
fossil_fatal("no such item: %d", blobid);
}
if( bag_find(&busy, blobid) ) fossil_fatal("delta loop");
blob_zero(&x);
db_column_blob(&q, 2, &x);
blob_uncompress(&x, &x);
if( db_column_type(&q,1)==SQLITE_INTEGER ){
bundle_extract_item(db_column_int(&q,1), &basis);
blob_delta_apply(&basis, &x, pOut);
blob_reset(&basis);
blob_reset(&x);
}else if( db_column_type(&q,1)==SQLITE_TEXT ){
int rid = db_int(0, "SELECT rid FROM blob WHERE uuid=%Q",
db_column_text(&q,1));
if( rid==0 ){
fossil_fatal("cannot find delta basis %s", db_column_text(&q,1));
}
content_get(rid, &basis);
db_column_blob(&q, 2, &x);
blob_delta_apply(&basis, &x, pOut);
blob_reset(&basis);
blob_reset(&x);
}else{
*pOut = x;
}
blob_zero(&h1);
db_column_blob(&q, 0, &h1);
sha1sum_blob(pOut, &h2);
if( blob_compare(&h1, &h2)!=0 ){
fossil_fatal("SHA1 hash mismatch - wanted %s, got %s",
blob_str(&h1), blob_str(&h2));
}
blob_reset(&h1);
blob_reset(&h2);
bag_remove(&busy, blobid);
db_finalize(&q);
}
/*
** Load the record identify by rid. Make sure we can reproduce it
** without error.
**
** Panic if anything goes wrong. If this procedure returns it means
** that everything is OK.
*/
static void verify_rid(int rid){
Blob uuid, hash, content;
if( content_size(rid, 0)<0 ){
return; /* No way to verify phantoms */
}
blob_zero(&uuid);
db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d", rid);
if( blob_size(&uuid)!=UUID_SIZE ){
fossil_fatal("not a valid rid: %d", rid);
}
if( content_get(rid, &content) ){
sha1sum_blob(&content, &hash);
blob_reset(&content);
if( blob_compare(&uuid, &hash) ){
fossil_fatal("hash of rid %d (%b) does not match its uuid (%b)",
rid, &hash, &uuid);
}
blob_reset(&hash);
}
blob_reset(&uuid);
}
/*
** Insert an artifact into the BLOB table if it isn't there already.
** If zMark is not zero, create a cross-reference from that mark back
** to the newly inserted artifact.
**
** If saveUuid is true, then pContent is a commit record. Record its
** UUID in gg.zPrevCheckin.
*/
static int fast_insert_content(Blob *pContent, const char *zMark, int saveUuid){
Blob hash;
Blob cmpr;
int rid;
sha1sum_blob(pContent, &hash);
rid = db_int(0, "SELECT rid FROM blob WHERE uuid=%B", &hash);
if( rid==0 ){
static Stmt ins;
db_static_prepare(&ins,
"INSERT INTO blob(uuid, size, content) VALUES(:uuid, :size, :content)"
);
db_bind_text(&ins, ":uuid", blob_str(&hash));
db_bind_int(&ins, ":size", gg.nData);
blob_compress(pContent, &cmpr);
db_bind_blob(&ins, ":content", &cmpr);
db_step(&ins);
db_reset(&ins);
blob_reset(&cmpr);
rid = db_last_insert_rowid();
}
if( zMark ){
db_multi_exec(
"INSERT OR IGNORE INTO xmark(tname, trid, tuuid)"
"VALUES(%Q,%d,%B)",
zMark, rid, &hash
);
db_multi_exec(
"INSERT OR IGNORE INTO xmark(tname, trid, tuuid)"
"VALUES(%B,%d,%B)",
&hash, rid, &hash
);
}
if( saveUuid ){
fossil_free(gg.zPrevCheckin);
gg.zPrevCheckin = fossil_strdup(blob_str(&hash));
}
blob_reset(&hash);
return rid;
}
/*
** COMMAND: test-integrity ?OPTIONS?
**
** Verify that all content can be extracted from the BLOB table correctly.
** If the BLOB table is correct, then the repository can always be
** successfully reconstructed using "fossil rebuild".
**
** Options:
**
** --parse Parse all manifests, wikis, tickets, events, and
** so forth, reporting any errors found.
*/
void test_integrity(void){
Stmt q;
Blob content;
Blob cksum;
int n1 = 0;
int n2 = 0;
int nErr = 0;
int total;
int nCA = 0;
int anCA[10];
int bParse = find_option("parse",0,0)!=0;
db_find_and_open_repository(OPEN_ANY_SCHEMA, 2);
memset(anCA, 0, sizeof(anCA));
/* Make sure no public artifact is a delta from a private artifact */
db_prepare(&q,
"SELECT "
" rid, (SELECT uuid FROM blob WHERE rid=delta.rid),"
" srcid, (SELECT uuid FROM blob WHERE rid=delta.srcid)"
" FROM delta"
" WHERE srcid in private AND rid NOT IN private"
);
while( db_step(&q)==SQLITE_ROW ){
int rid = db_column_int(&q, 0);
const char *zId = db_column_text(&q, 1);
int srcid = db_column_int(&q, 2);
const char *zSrc = db_column_text(&q, 3);
fossil_print(
"public artifact %S (%d) is a delta from private artifact %S (%d)\n",
zId, rid, zSrc, srcid
);
nErr++;
}
db_finalize(&q);
db_prepare(&q, "SELECT rid, uuid, size FROM blob ORDER BY rid");
total = db_int(0, "SELECT max(rid) FROM blob");
while( db_step(&q)==SQLITE_ROW ){
int rid = db_column_int(&q, 0);
const char *zUuid = db_column_text(&q, 1);
int size = db_column_int(&q, 2);
n1++;
fossil_print(" %d/%d\r", n1, total);
fflush(stdout);
if( size<0 ){
fossil_print("skip phantom %d %s\n", rid, zUuid);
continue; /* Ignore phantoms */
}
content_get(rid, &content);
if( blob_size(&content)!=size ){
fossil_print("size mismatch on artifact %d: wanted %d but got %d\n",
rid, size, blob_size(&content));
nErr++;
}
sha1sum_blob(&content, &cksum);
if( fossil_strcmp(blob_str(&cksum), zUuid)!=0 ){
fossil_print("checksum mismatch on artifact %d: wanted %s but got %s\n",
rid, zUuid, blob_str(&cksum));
nErr++;
}
if( bParse && looks_like_control_artifact(&content) ){
Blob err;
int i, n;
char *z;
Manifest *p;
char zFirstLine[400];
blob_zero(&err);
z = blob_buffer(&content);
n = blob_size(&content);
for(i=0; i<n && z[i] && z[i]!='\n' && i<sizeof(zFirstLine)-1; i++){}
memcpy(zFirstLine, z, i);
zFirstLine[i] = 0;
p = manifest_parse(&content, 0, &err);
if( p==0 ){
fossil_print("manifest_parse failed for %s:\n%s\n",
blob_str(&cksum), blob_str(&err));
if( strncmp(blob_str(&err), "line 1:", 7)==0 ){
fossil_print("\"%s\"\n", zFirstLine);
}
}else{
anCA[p->type]++;
manifest_destroy(p);
nCA++;
}
blob_reset(&err);
}else{
blob_reset(&content);
}
blob_reset(&cksum);
n2++;
}
db_finalize(&q);
fossil_print("%d non-phantom blobs (out of %d total) checked: %d errors\n",
n2, n1, nErr);
if( bParse ){
const char *azType[] = { 0, "manifest", "cluster", "control", "wiki",
"ticket", "attachment", "event" };
int i;
fossil_print("%d total control artifacts\n", nCA);
for(i=1; i<count(azType); i++){
if( anCA[i] ) fossil_print(" %d %ss\n", anCA[i], azType[i]);
}
}
}
/*
** Write content into the database. Return the record ID. If the
** content is already in the database, just return the record ID.
**
** If srcId is specified, then pBlob is delta content from
** the srcId record. srcId might be a phantom.
**
** pBlob is normally uncompressed text. But if nBlob>0 then the
** pBlob value has already been compressed and nBlob is its uncompressed
** size. If nBlob>0 then zUuid must be valid.
**
** zUuid is the UUID of the artifact, if it is specified. When srcId is
** specified then zUuid must always be specified. If srcId is zero,
** and zUuid is zero then the correct zUuid is computed from pBlob.
**
** If the record already exists but is a phantom, the pBlob content
** is inserted and the phatom becomes a real record.
**
** The original content of pBlob is not disturbed. The caller continues
** to be responsible for pBlob. This routine does *not* take over
** responsibility for freeing pBlob.
*/
int content_put_ex(
Blob *pBlob, /* Content to add to the repository */
const char *zUuid, /* SHA1 hash of reconstructed pBlob */
int srcId, /* pBlob is a delta from this entry */
int nBlob, /* pBlob is compressed. Original size is this */
int isPrivate /* The content should be marked private */
){
int size;
int rid;
Stmt s1;
Blob cmpr;
Blob hash;
int markAsUnclustered = 0;
int isDephantomize = 0;
assert( g.repositoryOpen );
assert( pBlob!=0 );
assert( srcId==0 || zUuid!=0 );
if( zUuid==0 ){
assert( nBlob==0 );
sha1sum_blob(pBlob, &hash);
}else{
blob_init(&hash, zUuid, -1);
}
if( nBlob ){
size = nBlob;
}else{
size = blob_size(pBlob);
if( srcId ){
size = delta_output_size(blob_buffer(pBlob), size);
}
}
db_begin_transaction();
/* Check to see if the entry already exists and if it does whether
** or not the entry is a phantom
*/
db_prepare(&s1, "SELECT rid, size FROM blob WHERE uuid=%B", &hash);
if( db_step(&s1)==SQLITE_ROW ){
rid = db_column_int(&s1, 0);
if( db_column_int(&s1, 1)>=0 || pBlob==0 ){
/* Either the entry is not a phantom or it is a phantom but we
** have no data with which to dephantomize it. In either case,
** there is nothing for us to do other than return the RID. */
db_finalize(&s1);
db_end_transaction(0);
return rid;
}
}else{
rid = 0; /* No entry with the same UUID currently exists */
markAsUnclustered = 1;
}
db_finalize(&s1);
/* Construct a received-from ID if we do not already have one */
if( g.rcvid==0 ){
db_multi_exec(
"INSERT INTO rcvfrom(uid, mtime, nonce, ipaddr)"
"VALUES(%d, julianday('now'), %Q, %Q)",
g.userUid, g.zNonce, g.zIpAddr
);
g.rcvid = db_last_insert_rowid();
}
if( nBlob ){
cmpr = pBlob[0];
}else{
blob_compress(pBlob, &cmpr);
}
if( rid>0 ){
/* We are just adding data to a phantom */
db_prepare(&s1,
"UPDATE blob SET rcvid=%d, size=%d, content=:data WHERE rid=%d",
g.rcvid, size, rid
);
db_bind_blob(&s1, ":data", &cmpr);
db_exec(&s1);
db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
if( srcId==0 || content_is_available(srcId) ){
isDephantomize = 1;
content_mark_available(rid);
}
}else{
/* We are creating a new entry */
db_prepare(&s1,
"INSERT INTO blob(rcvid,size,uuid,content)"
"VALUES(%d,%d,'%b',:data)",
g.rcvid, size, &hash
);
db_bind_blob(&s1, ":data", &cmpr);
db_exec(&s1);
rid = db_last_insert_rowid();
if( !pBlob ){
db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
}
if( g.markPrivate || isPrivate ){
db_multi_exec("INSERT INTO private VALUES(%d)", rid);
markAsUnclustered = 0;
}
}
if( nBlob==0 ) blob_reset(&cmpr);
/* If the srcId is specified, then the data we just added is
** really a delta. Record this fact in the delta table.
*/
if( srcId ){
db_multi_exec("REPLACE INTO delta(rid,srcid) VALUES(%d,%d)", rid, srcId);
}
if( !isDephantomize && bag_find(&contentCache.missing, rid) &&
(srcId==0 || content_is_available(srcId)) ){
content_mark_available(rid);
}
if( isDephantomize ){
after_dephantomize(rid, 0);
}
/* Add the element to the unclustered table if has never been
** previously seen.
*/
if( markAsUnclustered ){
db_multi_exec("INSERT OR IGNORE INTO unclustered VALUES(%d)", rid);
}
/* Finish the transaction and cleanup */
db_finalize(&s1);
db_end_transaction(0);
blob_reset(&hash);
/* Make arrangements to verify that the data can be recovered
** before we commit */
verify_before_commit(rid);
return rid;
}
/*
** Given the RID for a checkin, construct a tarball containing
** all files in that checkin
**
** If RID is for an object that is not a real manifest, then the
** resulting tarball contains a single file which is the RID
** object.
**
** If the RID object does not exist in the repository, then
** pTar is zeroed.
**
** zDir is a "synthetic" subdirectory which all files get
** added to as part of the tarball. It may be 0 or an empty string, in
** which case it is ignored. The intention is to create a tarball which
** politely expands into a subdir instead of filling your current dir
** with source files. For example, pass a UUID or "ProjectName".
**
*/
void tarball_of_checkin(int rid, Blob *pTar, const char *zDir){
Blob mfile, hash, file;
Manifest *pManifest;
ManifestFile *pFile;
Blob filename;
int nPrefix;
char *zName;
unsigned int mTime;
content_get(rid, &mfile);
if( blob_size(&mfile)==0 ){
blob_zero(pTar);
return;
}
blob_zero(&hash);
blob_zero(&filename);
if( zDir && zDir[0] ){
blob_appendf(&filename, "%s/", zDir);
}
nPrefix = blob_size(&filename);
pManifest = manifest_get(rid, CFTYPE_MANIFEST);
if( pManifest ){
mTime = (pManifest->rDate - 2440587.5)*86400.0;
tar_begin(mTime);
if( db_get_boolean("manifest", 0) ){
blob_append(&filename, "manifest", -1);
zName = blob_str(&filename);
tar_add_file(zName, &mfile, 0, mTime);
sha1sum_blob(&mfile, &hash);
blob_reset(&mfile);
blob_append(&hash, "\n", 1);
blob_resize(&filename, nPrefix);
blob_append(&filename, "manifest.uuid", -1);
zName = blob_str(&filename);
tar_add_file(zName, &hash, 0, mTime);
blob_reset(&hash);
}
manifest_file_rewind(pManifest);
while( (pFile = manifest_file_next(pManifest,0))!=0 ){
int fid = uuid_to_rid(pFile->zUuid, 0);
if( fid ){
content_get(fid, &file);
blob_resize(&filename, nPrefix);
blob_append(&filename, pFile->zName, -1);
zName = blob_str(&filename);
tar_add_file(zName, &file, manifest_file_mperm(pFile), mTime);
blob_reset(&file);
}
}
}else{
sha1sum_blob(&mfile, &hash);
blob_append(&filename, blob_str(&hash), 16);
zName = blob_str(&filename);
mTime = db_int64(0, "SELECT (julianday('now') - 2440587.5)*86400.0;");
tar_begin(mTime);
tar_add_file(zName, &mfile, 0, mTime);
}
manifest_destroy(pManifest);
blob_reset(&mfile);
blob_reset(&filename);
tar_finish(pTar);
}
/*
** COMMAND: test-integrity
**
** Verify that all content can be extracted from the BLOB table correctly.
** If the BLOB table is correct, then the repository can always be
** successfully reconstructed using "fossil rebuild".
*/
void test_integrity(void){
Stmt q;
Blob content;
Blob cksum;
int n1 = 0;
int n2 = 0;
int nErr = 0;
int total;
db_find_and_open_repository(OPEN_ANY_SCHEMA, 2);
/* Make sure no public artifact is a delta from a private artifact */
db_prepare(&q,
"SELECT "
" rid, (SELECT uuid FROM blob WHERE rid=delta.rid),"
" srcid, (SELECT uuid FROM blob WHERE rid=delta.srcid)"
" FROM delta"
" WHERE srcid in private AND rid NOT IN private"
);
while( db_step(&q)==SQLITE_ROW ){
int rid = db_column_int(&q, 0);
const char *zId = db_column_text(&q, 1);
int srcid = db_column_int(&q, 2);
const char *zSrc = db_column_text(&q, 3);
fossil_print(
"public artifact %S (%d) is a delta from private artifact %S (%d)\n",
zId, rid, zSrc, srcid
);
nErr++;
}
db_finalize(&q);
db_prepare(&q, "SELECT rid, uuid, size FROM blob ORDER BY rid");
total = db_int(0, "SELECT max(rid) FROM blob");
while( db_step(&q)==SQLITE_ROW ){
int rid = db_column_int(&q, 0);
const char *zUuid = db_column_text(&q, 1);
int size = db_column_int(&q, 2);
n1++;
fossil_print(" %d/%d\r", n1, total);
fflush(stdout);
if( size<0 ){
fossil_print("skip phantom %d %s\n", rid, zUuid);
continue; /* Ignore phantoms */
}
content_get(rid, &content);
if( blob_size(&content)!=size ){
fossil_print("size mismatch on artifact %d: wanted %d but got %d\n",
rid, size, blob_size(&content));
nErr++;
}
sha1sum_blob(&content, &cksum);
if( fossil_strcmp(blob_str(&cksum), zUuid)!=0 ){
fossil_print("checksum mismatch on artifact %d: wanted %s but got %s\n",
rid, zUuid, blob_str(&cksum));
nErr++;
}
blob_reset(&cksum);
blob_reset(&content);
n2++;
}
db_finalize(&q);
fossil_print("%d non-phantom blobs (out of %d total) checked: %d errors\n",
n2, n1, nErr);
}
/*
** There is a TEMP table bix(blobid,delta) containing a set of purgeitems
** that need to be transferred to the BLOB table. This routine does
** all items that have srcid=iSrc. The pBasis blob holds the content
** of the source document if iSrc>0.
*/
static void bundle_import_elements(int iSrc, Blob *pBasis, int isPriv){
Stmt q;
static Bag busy;
assert( pBasis!=0 || iSrc==0 );
if( iSrc>0 ){
if( bag_find(&busy, iSrc) ){
fossil_fatal("delta loop while uncompressing bundle artifacts");
}
bag_insert(&busy, iSrc);
}
db_prepare(&q,
"SELECT uuid, data, bblob.delta, bix.blobid"
" FROM bix, bblob"
" WHERE bix.delta=%d"
" AND bix.blobid=bblob.blobid;",
iSrc
);
while( db_step(&q)==SQLITE_ROW ){
Blob h1, h2, c1, c2;
int rid;
blob_zero(&h1);
db_column_blob(&q, 0, &h1);
blob_zero(&c1);
db_column_blob(&q, 1, &c1);
blob_uncompress(&c1, &c1);
blob_zero(&c2);
if( db_column_type(&q,2)==SQLITE_TEXT && db_column_bytes(&q,2)==40 ){
Blob basis;
rid = db_int(0,"SELECT rid FROM blob WHERE uuid=%Q",
db_column_text(&q,2));
content_get(rid, &basis);
blob_delta_apply(&basis, &c1, &c2);
blob_reset(&basis);
blob_reset(&c1);
}else if( pBasis ){
blob_delta_apply(pBasis, &c1, &c2);
blob_reset(&c1);
}else{
c2 = c1;
}
sha1sum_blob(&c2, &h2);
if( blob_compare(&h1, &h2)!=0 ){
fossil_fatal("SHA1 hash mismatch - wanted %s, got %s",
blob_str(&h1), blob_str(&h2));
}
blob_reset(&h2);
rid = content_put_ex(&c2, blob_str(&h1), 0, 0, isPriv);
if( rid==0 ){
fossil_fatal("%s", g.zErrMsg);
}else{
if( !isPriv ) content_make_public(rid);
content_get(rid, &c1);
manifest_crosslink(rid, &c1, MC_NO_ERRORS);
db_multi_exec("INSERT INTO got(rid) VALUES(%d)",rid);
}
bundle_import_elements(db_column_int(&q,3), &c2, isPriv);
blob_reset(&c2);
}
db_finalize(&q);
if( iSrc>0 ) bag_remove(&busy, iSrc);
}