Пример #1
0
/*
** COMMAND: test-compress
*/
void compress_cmd(void){
  Blob f;
  if( g.argc!=4 ) usage("INPUTFILE OUTPUTFILE");
  blob_read_from_file(&f, g.argv[2]);
  blob_compress(&f, &f);
  blob_write_to_file(&f, g.argv[3]);
}
Пример #2
0
/*
** Implement the "fossil bundle append BUNDLE FILE..." command.  Add
** the named files into the BUNDLE.  Create the BUNDLE if it does not
** alraedy exist.
*/
static void bundle_append_cmd(void){
  Blob content, hash;
  int i;
  Stmt q;

  verify_all_options();
  bundle_attach_file(g.argv[3], "b1", 1);
  db_prepare(&q,
    "INSERT INTO bblob(blobid, uuid, sz, delta, data, notes) "
    "VALUES(NULL, $uuid, $sz, NULL, $data, $filename)");
  db_begin_transaction();
  for(i=4; i<g.argc; i++){
    int sz;
    blob_read_from_file(&content, g.argv[i]);
    sz = blob_size(&content);
    sha1sum_blob(&content, &hash);
    blob_compress(&content, &content);
    db_bind_text(&q, "$uuid", blob_str(&hash));
    db_bind_int(&q, "$sz", sz);
    db_bind_blob(&q, "$data", &content);
    db_bind_text(&q, "$filename", g.argv[i]);
    db_step(&q);
    db_reset(&q);
    blob_reset(&content);
    blob_reset(&hash);
  }
  db_end_transaction(0);
  db_finalize(&q);
}
Пример #3
0
/*
** Change the storage of rid so that it is a delta of srcid.
**
** If rid is already a delta from some other place then no
** conversion occurs and this is a no-op unless force==1.
**
** Never generate a delta that carries a private artifact into a public
** artifact.  Otherwise, when we go to send the public artifact on a
** sync operation, the other end of the sync will never be able to receive
** the source of the delta.  It is OK to delta private->private and
** public->private and public->public.  Just no private->public delta.
**
** If srcid is a delta that depends on rid, then srcid is
** converted to undeltaed text.
**
** If either rid or srcid contain less than 50 bytes, or if the
** resulting delta does not achieve a compression of at least 25% 
** the rid is left untouched.
**
** Return 1 if a delta is made and 0 if no delta occurs.
*/
int content_deltify(int rid, int srcid, int force){
  int s;
  Blob data, src, delta;
  Stmt s1, s2;
  int rc = 0;

  if( srcid==rid ) return 0;
  if( !force && findSrcid(rid)>0 ) return 0;
  if( content_is_private(srcid) && !content_is_private(rid) ){
    return 0;
  }
  s = srcid;
  while( (s = findSrcid(s))>0 ){
    if( s==rid ){
      content_undelta(srcid);
      break;
    }
  }
  content_get(srcid, &src);
  if( blob_size(&src)<50 ){
    blob_reset(&src);
    return 0;
  }
  content_get(rid, &data);
  if( blob_size(&data)<50 ){
    blob_reset(&src);
    blob_reset(&data);
    return 0;
  }
  blob_delta_create(&src, &data, &delta);
  if( blob_size(&delta) <= blob_size(&data)*0.75 ){
    blob_compress(&delta, &delta);
    db_prepare(&s1, "UPDATE blob SET content=:data WHERE rid=%d", rid);
    db_prepare(&s2, "REPLACE INTO delta(rid,srcid)VALUES(%d,%d)", rid, srcid);
    db_bind_blob(&s1, ":data", &delta);
    db_begin_transaction();
    db_exec(&s1);
    db_exec(&s2);
    db_end_transaction(0);
    db_finalize(&s1);
    db_finalize(&s2);
    verify_before_commit(rid);
    rc = 1;
  }
  blob_reset(&src);
  blob_reset(&data);
  blob_reset(&delta);
  return rc;
}
Пример #4
0
/*
** Make sure the content at rid is the original content and is not a
** delta.
*/
void content_undelta(int rid){
  if( findSrcid(rid)>0 ){
    Blob x;
    if( content_get(rid, &x) ){
      Stmt s;
      db_prepare(&s, "UPDATE blob SET content=:c, size=%d WHERE rid=%d",
                     blob_size(&x), rid);
      blob_compress(&x, &x);
      db_bind_blob(&s, ":c", &x);
      db_exec(&s);
      db_finalize(&s);
      blob_reset(&x);
      db_multi_exec("DELETE FROM delta WHERE rid=%d", rid);
    }
  }
}
Пример #5
0
/*
** COMMAND: test-cycle-compress
**
** Compress and uncompress each file named on the command line.
** Verify that the original content is recovered.
*/
void test_cycle_compress(void){
  int i;
  Blob b1, b2, b3;
  for(i=2; i<g.argc; i++){
    blob_read_from_file(&b1, g.argv[i]);
    blob_compress(&b1, &b2);
    blob_uncompress(&b2, &b3);
    if( blob_compare(&b1, &b3) ){
      fossil_fatal("compress/uncompress cycle failed for %s", g.argv[i]);
    }
    blob_reset(&b1);
    blob_reset(&b2);
    blob_reset(&b3);
  }
  fossil_print("ok\n");
}
Пример #6
0
/*
** Insert an artifact into the BLOB table if it isn't there already.
** If zMark is not zero, create a cross-reference from that mark back
** to the newly inserted artifact.
**
** If saveUuid is true, then pContent is a commit record.  Record its
** UUID in gg.zPrevCheckin.
*/
static int fast_insert_content(Blob *pContent, const char *zMark, int saveUuid){
  Blob hash;
  Blob cmpr;
  int rid;

  sha1sum_blob(pContent, &hash);
  rid = db_int(0, "SELECT rid FROM blob WHERE uuid=%B", &hash);
  if( rid==0 ){
    static Stmt ins;
    db_static_prepare(&ins,
        "INSERT INTO blob(uuid, size, content) VALUES(:uuid, :size, :content)"
    );
    db_bind_text(&ins, ":uuid", blob_str(&hash));
    db_bind_int(&ins, ":size", gg.nData);
    blob_compress(pContent, &cmpr);
    db_bind_blob(&ins, ":content", &cmpr);
    db_step(&ins);
    db_reset(&ins);
    blob_reset(&cmpr);
    rid = db_last_insert_rowid();
  }
  if( zMark ){
    db_multi_exec(
        "INSERT OR IGNORE INTO xmark(tname, trid, tuuid)"
        "VALUES(%Q,%d,%B)",
        zMark, rid, &hash
    );
    db_multi_exec(
        "INSERT OR IGNORE INTO xmark(tname, trid, tuuid)"
        "VALUES(%B,%d,%B)",
        &hash, rid, &hash
    );
  }
  if( saveUuid ){
    fossil_free(gg.zPrevCheckin);
    gg.zPrevCheckin = fossil_strdup(blob_str(&hash));
  }
  blob_reset(&hash);
  return rid;
}
Пример #7
0
/*
** Write content into the database.  Return the record ID.  If the
** content is already in the database, just return the record ID.
**
** If srcId is specified, then pBlob is delta content from
** the srcId record.  srcId might be a phantom.  
**
** pBlob is normally uncompressed text.  But if nBlob>0 then the
** pBlob value has already been compressed and nBlob is its uncompressed
** size.  If nBlob>0 then zUuid must be valid.
**
** zUuid is the UUID of the artifact, if it is specified.  When srcId is
** specified then zUuid must always be specified.  If srcId is zero,
** and zUuid is zero then the correct zUuid is computed from pBlob.
**
** If the record already exists but is a phantom, the pBlob content
** is inserted and the phatom becomes a real record.
**
** The original content of pBlob is not disturbed.  The caller continues
** to be responsible for pBlob.  This routine does *not* take over
** responsibility for freeing pBlob.
*/
int content_put_ex(
  Blob *pBlob,              /* Content to add to the repository */
  const char *zUuid,        /* SHA1 hash of reconstructed pBlob */
  int srcId,                /* pBlob is a delta from this entry */
  int nBlob,                /* pBlob is compressed. Original size is this */
  int isPrivate             /* The content should be marked private */
){
  int size;
  int rid;
  Stmt s1;
  Blob cmpr;
  Blob hash;
  int markAsUnclustered = 0;
  int isDephantomize = 0;
  
  assert( g.repositoryOpen );
  assert( pBlob!=0 );
  assert( srcId==0 || zUuid!=0 );
  if( zUuid==0 ){
    assert( nBlob==0 );
    sha1sum_blob(pBlob, &hash);
  }else{
    blob_init(&hash, zUuid, -1);
  }
  if( nBlob ){
    size = nBlob;
  }else{
    size = blob_size(pBlob);
    if( srcId ){
      size = delta_output_size(blob_buffer(pBlob), size);
    }
  }
  db_begin_transaction();

  /* Check to see if the entry already exists and if it does whether
  ** or not the entry is a phantom
  */
  db_prepare(&s1, "SELECT rid, size FROM blob WHERE uuid=%B", &hash);
  if( db_step(&s1)==SQLITE_ROW ){
    rid = db_column_int(&s1, 0);
    if( db_column_int(&s1, 1)>=0 || pBlob==0 ){
      /* Either the entry is not a phantom or it is a phantom but we
      ** have no data with which to dephantomize it.  In either case,
      ** there is nothing for us to do other than return the RID. */
      db_finalize(&s1);
      db_end_transaction(0);
      return rid;
    }
  }else{
    rid = 0;  /* No entry with the same UUID currently exists */
    markAsUnclustered = 1;
  }
  db_finalize(&s1);

  /* Construct a received-from ID if we do not already have one */
  if( g.rcvid==0 ){
    db_multi_exec(
       "INSERT INTO rcvfrom(uid, mtime, nonce, ipaddr)"
       "VALUES(%d, julianday('now'), %Q, %Q)",
       g.userUid, g.zNonce, g.zIpAddr
    );
    g.rcvid = db_last_insert_rowid();
  }

  if( nBlob ){
    cmpr = pBlob[0];
  }else{
    blob_compress(pBlob, &cmpr);
  }
  if( rid>0 ){
    /* We are just adding data to a phantom */
    db_prepare(&s1,
      "UPDATE blob SET rcvid=%d, size=%d, content=:data WHERE rid=%d",
       g.rcvid, size, rid
    );
    db_bind_blob(&s1, ":data", &cmpr);
    db_exec(&s1);
    db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
    if( srcId==0 || content_is_available(srcId) ){
      isDephantomize = 1;
      content_mark_available(rid);
    }
  }else{
    /* We are creating a new entry */
    db_prepare(&s1,
      "INSERT INTO blob(rcvid,size,uuid,content)"
      "VALUES(%d,%d,'%b',:data)",
       g.rcvid, size, &hash
    );
    db_bind_blob(&s1, ":data", &cmpr);
    db_exec(&s1);
    rid = db_last_insert_rowid();
    if( !pBlob ){
      db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
    }
    if( g.markPrivate || isPrivate ){
      db_multi_exec("INSERT INTO private VALUES(%d)", rid);
      markAsUnclustered = 0;
    }
  }
  if( nBlob==0 ) blob_reset(&cmpr);

  /* If the srcId is specified, then the data we just added is
  ** really a delta.  Record this fact in the delta table.
  */
  if( srcId ){
    db_multi_exec("REPLACE INTO delta(rid,srcid) VALUES(%d,%d)", rid, srcId);
  }
  if( !isDephantomize && bag_find(&contentCache.missing, rid) && 
      (srcId==0 || content_is_available(srcId)) ){
    content_mark_available(rid);
  }
  if( isDephantomize ){
    after_dephantomize(rid, 0);
  }
  
  /* Add the element to the unclustered table if has never been
  ** previously seen.
  */
  if( markAsUnclustered ){
    db_multi_exec("INSERT OR IGNORE INTO unclustered VALUES(%d)", rid);
  }

  /* Finish the transaction and cleanup */
  db_finalize(&s1);
  db_end_transaction(0);
  blob_reset(&hash);

  /* Make arrangements to verify that the data can be recovered
  ** before we commit */
  verify_before_commit(rid);
  return rid;
}
Пример #8
0
/* fossil bundle export BUNDLE ?OPTIONS?
**
** OPTIONS:
**   --branch BRANCH --from TAG --to TAG
**   --checkin TAG
**   --standalone
*/
static void bundle_export_cmd(void){
  int bStandalone = find_option("standalone",0,0)!=0;
  int mnToBundle;   /* Minimum RID in the bundle */
  Stmt q;

  /* Decode the arguments (like --branch) that specify which artifacts
  ** should be in the bundle */
  db_multi_exec("CREATE TEMP TABLE tobundle(rid INTEGER PRIMARY KEY);");
  subtree_from_arguments("tobundle");
  find_checkin_associates("tobundle", 0);
  verify_all_options();
  describe_artifacts("IN tobundle");

  if( g.argc!=4 ) usage("export BUNDLE ?OPTIONS?");
  /* Create the new bundle */
  bundle_attach_file(g.argv[3], "b1", 1);
  db_begin_transaction();

  /* Add 'mtime' and 'project-code' entries to the bconfig table */
  db_multi_exec(
    "INSERT INTO bconfig(bcname,bcvalue)"
    " VALUES('mtime',datetime('now'));"
  );
  db_multi_exec(
    "INSERT INTO bconfig(bcname,bcvalue)"
    " SELECT name, value FROM config"
    "  WHERE name IN ('project-code');"
  );

  /* Directly copy content from the repository into the bundle as long
  ** as the repository content is a delta from some other artifact that
  ** is also in the bundle.
  */
  db_multi_exec(
    "REPLACE INTO bblob(blobid,uuid,sz,delta,data,notes) "
    " SELECT"
    "   tobundle.rid,"
    "   blob.uuid,"
    "   blob.size,"
    "   delta.srcid,"
    "   blob.content,"
    "   (SELECT summary FROM description WHERE rid=blob.rid)"
    " FROM tobundle, blob, delta"
    " WHERE blob.rid=tobundle.rid"
    "   AND delta.rid=tobundle.rid"
    "   AND delta.srcid IN tobundle;"
  );

  /* For all the remaining artifacts, we need to construct their deltas
  ** manually.
  */
  mnToBundle = db_int(0,"SELECT min(rid) FROM tobundle");
  db_prepare(&q,
     "SELECT rid FROM tobundle"
     " WHERE rid NOT IN (SELECT blobid FROM bblob)"
     " ORDER BY +rid;"
  );
  while( db_step(&q)==SQLITE_ROW ){
    Blob content;
    int rid = db_column_int(&q,0);
    int deltaFrom = 0;

    /* Get the raw, uncompressed content of the artifact into content */
    content_get(rid, &content);

    /* Try to find another artifact, not within the bundle, that is a
    ** plausible candidate for being a delta basis for the content.  Set
    ** deltaFrom to the RID of that other artifact.  Leave deltaFrom set
    ** to zero if the content should not be delta-compressed
    */
    if( !bStandalone ){
      if( db_exists("SELECT 1 FROM plink WHERE cid=%d",rid) ){
        deltaFrom = db_int(0,
           "SELECT max(cid) FROM plink"
           " WHERE cid<%d", mnToBundle);
      }else{
        deltaFrom = db_int(0,
           "SELECT max(fid) FROM mlink"
           " WHERE fnid=(SELECT fnid FROM mlink WHERE fid=%d)"
           "   AND fid<%d", rid, mnToBundle);
      }
    }

    /* Try to insert the insert the artifact as a delta
    */
    if( deltaFrom ){
      Blob basis, delta;
      content_get(deltaFrom, &basis);
      blob_delta_create(&basis, &content, &delta);
      if( blob_size(&delta)>0.9*blob_size(&content) ){
        deltaFrom = 0;
      }else{
        Stmt ins;
        blob_compress(&delta, &delta);
        db_prepare(&ins,
          "REPLACE INTO bblob(blobid,uuid,sz,delta,data,notes)"
          " SELECT %d, uuid, size, (SELECT uuid FROM blob WHERE rid=%d),"
          "  :delta, (SELECT summary FROM description WHERE rid=blob.rid)"
          "  FROM blob WHERE rid=%d", rid, deltaFrom, rid);
        db_bind_blob(&ins, ":delta", &delta);
        db_step(&ins);
        db_finalize(&ins);
      }
      blob_reset(&basis);
      blob_reset(&delta);
    }

    /* If unable to insert the artifact as a delta, insert full-text */
    if( deltaFrom==0 ){
      Stmt ins;
      blob_compress(&content, &content);
      db_prepare(&ins,
        "REPLACE INTO bblob(blobid,uuid,sz,delta,data,notes)"
        " SELECT rid, uuid, size, NULL, :content,"
        "        (SELECT summary FROM description WHERE rid=blob.rid)"
          " FROM blob WHERE rid=%d", rid);
      db_bind_blob(&ins, ":content", &content);
      db_step(&ins);
      db_finalize(&ins);
    }
    blob_reset(&content);
  }
  db_finalize(&q);

  db_end_transaction(0);
}