void
hmac_sha1 ( guint8 *hmac, const guint8 *key, guint lk, const guint8 *data, guint ld )
{
GChecksum *sha1;
guint i;
gsize dl;
guint8 k_buf[SHA_DIGESTSIZE];
guint8 buf[SHA_BLOCKSIZE];
guint8 isha[SHA_DIGESTSIZE];
const guint8 *k;
sha1 = g_checksum_new ( G_CHECKSUM_SHA1 );
/* make sure key isn't too long */
if ( lk > SHA_BLOCKSIZE ) {
g_checksum_reset ( sha1 );
g_checksum_update ( sha1, key, lk );
dl = SHA_DIGESTSIZE;
g_checksum_get_digest ( sha1, k_buf, &dl);
lk = SHA_DIGESTSIZE;
k = k_buf;
} else {
k = key;
}
/**** Inner digest ****/
g_checksum_reset ( sha1 );
/* Pad the key for inner digest */
for ( i=0; i < lk; ++i ) buf[i] = k[i] ^ 0x36;
for ( i=lk; i < SHA_BLOCKSIZE; ++i ) buf[i] = 0x36;
g_checksum_update ( sha1, buf, SHA_BLOCKSIZE );
g_checksum_update ( sha1, data, ld );
dl = SHA_DIGESTSIZE;
g_checksum_get_digest ( sha1, isha, &dl);
/**** Outer digest ****/
g_checksum_reset ( sha1 );
/* Pad the key for outer digest */
for ( i=0; i < lk; ++i ) buf[i] = k[i] ^ 0x5C;
for ( i=lk; i < SHA_BLOCKSIZE; ++i ) buf[i] = 0x5C;
g_checksum_update ( sha1, buf, SHA_BLOCKSIZE );
g_checksum_update ( sha1, isha, SHA_DIGESTSIZE );
/* copy over the final result */
dl = SHA_DIGESTSIZE;
g_checksum_get_digest ( sha1, hmac, &dl);
/* cleanup */
g_checksum_free ( sha1 );
}
sc_bool sc_link_calculate_checksum(const sc_stream *stream, sc_check_sum *check_sum)
{
sc_char buffer[1024];
sc_uint32 data_read;
const gchar *result = 0;
GChecksum *checksum = g_checksum_new(SC_DEFAULT_CHECKSUM);
g_assert(stream != 0);
g_assert(check_sum != 0);
g_checksum_reset(checksum);
sc_stream_seek(stream, SC_STREAM_SEEK_SET, 0);
while (sc_stream_eof(stream) == SC_FALSE)
{
if (sc_stream_read_data(stream, buffer, 1024, &data_read) == SC_RESULT_ERROR)
{
g_checksum_free(checksum);
return SC_FALSE;
}
g_checksum_update(checksum, (guchar*)buffer, data_read);
}
// store results
check_sum->len = g_checksum_type_get_length(SC_DEFAULT_CHECKSUM);
result = g_checksum_get_string(checksum);
memcpy(&(check_sum->data[0]), result, check_sum->len);
g_checksum_free(checksum);
sc_stream_seek(stream, SC_STREAM_SEEK_SET, 0);
return SC_TRUE;
}
/* See RFC2069, 2.1.2 */
static gchar *
auth_digest_compute_response_md5 (const gchar * method, const gchar * realm,
const gchar * username, const gchar * password, const gchar * uri,
const gchar * nonce)
{
gchar hex_a1[33] = { 0, };
gchar hex_a2[33] = { 0, };
GChecksum *md5_context = g_checksum_new (G_CHECKSUM_MD5);
const gchar *digest_string;
gchar *response;
/* Compute A1 */
g_checksum_update (md5_context, (const guchar *) username, strlen (username));
g_checksum_update (md5_context, (const guchar *) ":", 1);
g_checksum_update (md5_context, (const guchar *) realm, strlen (realm));
g_checksum_update (md5_context, (const guchar *) ":", 1);
g_checksum_update (md5_context, (const guchar *) password, strlen (password));
digest_string = g_checksum_get_string (md5_context);
g_assert (strlen (digest_string) == 32);
memcpy (hex_a1, digest_string, 32);
g_checksum_reset (md5_context);
/* compute A2 */
g_checksum_update (md5_context, (const guchar *) method, strlen (method));
g_checksum_update (md5_context, (const guchar *) ":", 1);
g_checksum_update (md5_context, (const guchar *) uri, strlen (uri));
digest_string = g_checksum_get_string (md5_context);
g_assert (strlen (digest_string) == 32);
memcpy (hex_a2, digest_string, 32);
/* compute KD */
g_checksum_reset (md5_context);
g_checksum_update (md5_context, (const guchar *) hex_a1, strlen (hex_a1));
g_checksum_update (md5_context, (const guchar *) ":", 1);
g_checksum_update (md5_context, (const guchar *) nonce, strlen (nonce));
g_checksum_update (md5_context, (const guchar *) ":", 1);
g_checksum_update (md5_context, (const guchar *) hex_a2, 32);
response = g_strdup (g_checksum_get_string (md5_context));
g_checksum_free (md5_context);
return response;
}
static void
purple_sha1_hash_reset(PurpleHash *hash)
{
PurpleSHA1Hash *sha1_hash = PURPLE_SHA1_HASH(hash);
PurpleSHA1HashPrivate *priv = PURPLE_SHA1_HASH_GET_PRIVATE(sha1_hash);
g_return_if_fail(priv != NULL);
g_return_if_fail(priv->checksum != NULL);
g_checksum_reset(priv->checksum);
}
static void
_same_hash (const char *p0)
{
static gchar* memory[65536];
GString *prefix = g_string_new(p0);
gint64 counter;
GChecksum *c;
gchar num[64];
union {
guint8 b[32];
guint16 prefix;
} bin;
gsize binsize;
memset(&bin, 0, sizeof(bin));
counter = 0;
c = g_checksum_new(G_CHECKSUM_SHA256);
if (prefix && prefix->len > 0) {
/* pre-loads the memory with the prefix only */
g_checksum_update(c, (guint8*) prefix->str, prefix->len);
binsize = sizeof(bin.b);
g_checksum_get_digest(c, bin.b, &binsize);
memory[bin.prefix] = g_strdup(prefix->str);
}
for (;;) {
GString *gstr = g_string_new("");
if (prefix && prefix->len > 0)
g_string_append_len(gstr, prefix->str, prefix->len);
g_snprintf(num, sizeof(num), "%"G_GINT64_FORMAT, counter++);
g_string_append(gstr, num);
g_checksum_reset(c);
g_checksum_update(c, (guint8*) gstr->str, gstr->len);
binsize = sizeof(bin.b);
g_checksum_get_digest(c, bin.b, &binsize);
if (memory[bin.prefix]) {
g_print("%02X%02X %s %s\n", bin.b[0], bin.b[1],
memory[bin.prefix], gstr->str);
g_free(memory[bin.prefix]);
}
memory[bin.prefix] = g_string_free(gstr, FALSE);
}
g_checksum_free(c);
}
static void
_same_hash (const char *acct, const char *p0)
{
static gchar* memory[65536];
gint64 counter = 0;
union {
guint8 b[32];
guint16 prefix;
} bin;
memset(&bin, 0, sizeof(bin));
GChecksum *c = g_checksum_new(G_CHECKSUM_SHA256);
if (*p0) {
/* pre-loads the memory with the prefix only */
g_checksum_update(c, (guint8*) acct, strlen(acct));
g_checksum_update(c, (guint8*)"", 1);
g_checksum_update(c, (guint8*) p0, strlen(p0));
gsize binsize = sizeof(bin.b);
g_checksum_get_digest(c, bin.b, &binsize);
memory[bin.prefix] = g_strdup(p0);
}
for (;;) {
GString *gstr = g_string_new (p0);
g_string_append_printf (gstr, "%"G_GINT64_FORMAT, counter++);
g_checksum_reset(c);
g_checksum_update(c, (guint8*) acct, strlen(acct));
g_checksum_update(c, (guint8*)"", 1);
g_checksum_update(c, (guint8*) gstr->str, gstr->len);
gsize binsize = sizeof(bin.b);
g_checksum_get_digest(c, bin.b, &binsize);
if (memory[bin.prefix]) {
g_print("%02X%02X %s %s\n", bin.b[0], bin.b[1],
memory[bin.prefix], gstr->str);
g_free(memory[bin.prefix]);
}
memory[bin.prefix] = g_string_free(gstr, FALSE);
}
g_checksum_free(c);
}
char *jabber_sasl_cram_md5_response(session_t *s, char *challenge, const char *username, const char *password) {
char *digstr, *tmp, *retval;
gsize i, len, block_size = 64;
GChecksum *idigest = g_checksum_new(G_CHECKSUM_MD5);
GChecksum *odigest = g_checksum_new(G_CHECKSUM_MD5);
guchar *buf = g_malloc0(block_size);
guchar *pad = g_malloc0(block_size);
if (xstrlen(password) > block_size) {
gsize len = block_size;
g_checksum_update(idigest, (const guchar *)password, xstrlen(password));
g_checksum_get_digest(idigest, buf, &len);
g_checksum_reset(idigest);
} else
memcpy(buf, password, xstrlen(password));
/* ipad */
for (i = 0; i < block_size; i++)
pad[i] = 0x36 ^ buf[i];
g_checksum_update(idigest, pad, block_size);
/* opad */
for (i = 0; i < block_size; i++)
pad[i] = 0x5c ^ buf[i];
g_checksum_update(odigest, pad, block_size);
g_checksum_update(idigest, (const guchar *)challenge, xstrlen(challenge));
g_checksum_get_digest(idigest, buf, &len);
g_checksum_update(odigest, buf, len);
g_checksum_get_digest(odigest, buf, &len);
digstr = g_strdup(g_checksum_get_string(odigest));
g_checksum_free(idigest);
g_checksum_free(odigest);
g_free(buf);
g_free(pad);
tmp = g_strdup_printf("%s %s", username, digstr);
retval = base64_encode(tmp, xstrlen(tmp));
g_free(tmp);
g_free(digstr);
return retval;
}
int
moloch_hp_cb_on_message_begin (http_parser *parser)
{
HTTPInfo_t *http = parser->data;
MolochSession_t *session = http->session;
#ifdef HTTPDEBUG
LOG("HTTPDEBUG: which: %d", http->which);
#endif
http->inHeader &= ~(1 << http->which);
http->inValue &= ~(1 << http->which);
http->inBody &= ~(1 << http->which);
g_checksum_reset(http->checksum[http->which]);
if (pluginsCbs & MOLOCH_PLUGIN_HP_OMB)
moloch_plugins_cb_hp_omb(session, parser);
return 0;
}
gchar * _checksum_seg(sc_segment const * seg)
{
gchar * result = null_ptr;
gsize length = 0;
GChecksum * checksum = g_checksum_new(_checksum_type());
g_assert(seg);
if (checksum)
{
g_checksum_reset(checksum);
g_checksum_update(checksum, (guchar*)(&seg->elements[0]), SC_SEG_ELEMENTS_SIZE_BYTE);
length = _checksum_get_size();
result = g_new0(gchar, length);
g_checksum_get_digest(checksum, result, &length);
g_assert( length == _checksum_get_size() );
}
return result;
}
static SpruceSummaryReferences *
decode_references (const char *string)
{
SpruceSummaryReferences *references;
GMimeReferences *refs, *r;
unsigned char md5sum[16];
GChecksum *checksum;
guint32 i, n = 0;
size_t len = 16;
if (!(r = refs = g_mime_references_decode (string)))
return NULL;
while (r != NULL) {
r = r->next;
n++;
}
references = g_malloc (sizeof (SpruceSummaryReferences) + (sizeof (SpruceSummaryMessageID) * (n - 1)));
references->count = n;
checksum = g_checksum_new (G_CHECKSUM_MD5);
for (i = 0, r = refs; i < n; i++, r = r->next) {
g_checksum_update (checksum, r->msgid, strlen (r->msgid));
g_checksum_get_digest (checksum, md5sum, &len);
g_checksum_reset (checksum);
len = 16;
memcpy (references->references[i].id.hash, md5sum, sizeof (references->references[i].id.hash));
}
g_mime_references_clear (&refs);
g_checksum_free (checksum);
return references;
}
static void trigger(void)
{
guint8 *data;
gsize size, i;
if (step++ < N_TESTS)
{
size = g_random_int_range(0, 1024 * 1024);
data = g_malloc0(size);
for (i = 0; i < size; i++)
data[i] = g_random_int();
g_checksum_reset(checksum);
g_checksum_update(checksum, data, size);
sockmux_receiver_connect_filtered(receiver, step, receiver_cb, receiver);
sockmux_sender_send(sender, step, data, size);
g_free(data);
}
else
quit();
}
gpointer load_thumbnail_thread(gpointer user_data)
{
ThumbnailTask* task;
GChecksum* sum = g_checksum_new(G_CHECKSUM_MD5);
gchar* normal_path = g_build_filename(thumb_dir, "normal/00000000000000000000000000000000.png", NULL);
gchar* normal_basename = strrchr(normal_path, '/') + 1;
gchar* large_path = g_build_filename(thumb_dir, "large/00000000000000000000000000000000.png", NULL);
gchar* large_basename = strrchr(large_path, '/') + 1;
/* ensure thumbnail directories exists */
g_mkdir_with_parents(normal_path, 0700);
g_mkdir_with_parents(large_path, 0700);
for(;;)
{
G_LOCK(queue);
task = g_queue_pop_head(&loader_queue);
cur_loading = task;
if(G_LIKELY(task))
{
FmThumbnailRequest* req;
char* uri;
char* thumb_path;
const char* md5;
G_UNLOCK(queue);
uri = fm_path_to_uri(task->fi->path);
/* generate filename for the thumbnail */
g_checksum_update(sum, uri, -1);
md5 = g_checksum_get_string(sum); /* md5 sum of the URI */
task->uri = uri;
if (task->flags & LOAD_NORMAL)
{
memcpy( normal_basename, md5, 32 );
task->normal_path = normal_path;
}
if (task->flags & LOAD_LARGE)
{
memcpy( large_basename, md5, 32 );
task->large_path = large_path;
}
load_thumbnails(task);
g_checksum_reset(sum);
g_free(uri);
}
else /* no task is left in the loader_queue */
{
loader_thread_id = NULL;
G_UNLOCK(queue);
break;
}
}
g_free(normal_path);
g_free(large_path);
g_checksum_free(sum);
return NULL;
}
/*
* hmac_sha1:
* @key: The key
* @message: The message
*
* Given the key and message, compute the HMAC-SHA1 hash and return the base-64
* encoding of it. This is very geared towards OAuth, and as such both key and
* message must be NULL-terminated strings, and the result is base-64 encoded.
*/
char *
hmac_sha1 (const char *key, const char *message)
{
GChecksum *checksum;
char *real_key;
guchar ipad[SHA1_BLOCK_SIZE];
guchar opad[SHA1_BLOCK_SIZE];
guchar inner[SHA1_LENGTH];
guchar digest[SHA1_LENGTH];
gsize key_length, inner_length, digest_length;
int i;
g_return_val_if_fail (key, NULL);
g_return_val_if_fail (message, NULL);
checksum = g_checksum_new (G_CHECKSUM_SHA1);
/* If the key is longer than the block size, hash it first */
if (strlen (key) > SHA1_BLOCK_SIZE) {
guchar new_key[SHA1_LENGTH];
key_length = sizeof (new_key);
g_checksum_update (checksum, (guchar*)key, strlen (key));
g_checksum_get_digest (checksum, new_key, &key_length);
g_checksum_reset (checksum);
real_key = g_memdup (new_key, key_length);
} else {
real_key = g_strdup (key);
key_length = strlen (key);
}
/* Sanity check the length */
g_assert (key_length <= SHA1_BLOCK_SIZE);
/* Protect against use of the provided key by NULLing it */
key = NULL;
/* Stage 1 */
memset (ipad, 0, sizeof (ipad));
memset (opad, 0, sizeof (opad));
memcpy (ipad, real_key, key_length);
memcpy (opad, real_key, key_length);
/* Stage 2 and 5 */
for (i = 0; i < sizeof (ipad); i++) {
ipad[i] ^= 0x36;
opad[i] ^= 0x5C;
}
/* Stage 3 and 4 */
g_checksum_update (checksum, ipad, sizeof (ipad));
g_checksum_update (checksum, (guchar*)message, strlen (message));
inner_length = sizeof (inner);
g_checksum_get_digest (checksum, inner, &inner_length);
g_checksum_reset (checksum);
/* Stage 6 and 7 */
g_checksum_update (checksum, opad, sizeof (opad));
g_checksum_update (checksum, inner, inner_length);
digest_length = sizeof (digest);
g_checksum_get_digest (checksum, digest, &digest_length);
g_checksum_free (checksum);
g_free (real_key);
return g_base64_encode (digest, digest_length);
}
/**
* g_hmac_new:
* @digest_type: the desired type of digest
* @key: (array length=key_len): the key for the HMAC
* @key_len: the length of the keys
*
* Creates a new #GHmac, using the digest algorithm @digest_type.
* If the @digest_type is not known, %NULL is returned.
* A #GHmac can be used to compute the HMAC of a key and an
* arbitrary binary blob, using different hashing algorithms.
*
* A #GHmac works by feeding a binary blob through g_hmac_update()
* until the data is complete; the digest can then be extracted
* using g_hmac_get_string(), which will return the checksum as a
* hexadecimal string; or g_hmac_get_digest(), which will return a
* array of raw bytes. Once either g_hmac_get_string() or
* g_hmac_get_digest() have been called on a #GHmac, the HMAC
* will be closed and it won't be possible to call g_hmac_update()
* on it anymore.
*
* Support for digests of type %G_CHECKSUM_SHA512 has been added in GLib 2.42.
* Support for %G_CHECKSUM_SHA384 was added in GLib 2.52.
*
* Returns: the newly created #GHmac, or %NULL.
* Use g_hmac_unref() to free the memory allocated by it.
*
* Since: 2.30
*/
GHmac *
g_hmac_new (GChecksumType digest_type,
const guchar *key,
gsize key_len)
{
GChecksum *checksum;
GHmac *hmac;
guchar *buffer;
guchar *pad;
gsize i, len;
gsize block_size;
checksum = g_checksum_new (digest_type);
g_return_val_if_fail (checksum != NULL, NULL);
switch (digest_type)
{
case G_CHECKSUM_MD5:
case G_CHECKSUM_SHA1:
block_size = 64; /* RFC 2104 */
break;
case G_CHECKSUM_SHA256:
block_size = 64; /* RFC 4868 */
break;
case G_CHECKSUM_SHA384:
case G_CHECKSUM_SHA512:
block_size = 128; /* RFC 4868 */
break;
default:
g_return_val_if_reached (NULL);
}
hmac = g_slice_new0 (GHmac);
hmac->ref_count = 1;
hmac->digest_type = digest_type;
hmac->digesti = checksum;
hmac->digesto = g_checksum_new (digest_type);
buffer = g_alloca (block_size);
pad = g_alloca (block_size);
memset (buffer, 0, block_size);
/* If the key is too long, hash it */
if (key_len > block_size)
{
len = block_size;
g_checksum_update (hmac->digesti, key, key_len);
g_checksum_get_digest (hmac->digesti, buffer, &len);
g_checksum_reset (hmac->digesti);
}
/* Otherwise pad it with zeros */
else
{
memcpy (buffer, key, key_len);
}
/* First pad */
for (i = 0; i < block_size; i++)
pad[i] = 0x36 ^ buffer[i]; /* ipad value */
g_checksum_update (hmac->digesti, pad, block_size);
/* Second pad */
for (i = 0; i < block_size; i++)
pad[i] = 0x5c ^ buffer[i]; /* opad value */
g_checksum_update (hmac->digesto, pad, block_size);
return hmac;
}
/**
* Process the file that is not already in our local cache
* @param main_struct : main structure of the program
* @param meta is the meta data of the file to be processed (it does
* not contain any hashs at that point).
*/
static void process_big_file_not_in_cache(main_struct_t *main_struct, meta_data_t *meta)
{
GFile *a_file = NULL;
gchar *answer = NULL;
GFileInputStream *stream = NULL;
GError *error = NULL;
GList *hash_data_list = NULL;
GList *hdl_copy = NULL;
GList *saved_list = NULL;
hash_data_t *hash_data = NULL;
gssize read = 0;
guchar *buffer = NULL;
GChecksum *checksum = NULL;
guint8 *a_hash = NULL;
gsize digest_len = HASH_LEN;
gsize read_bytes = 0;
a_clock_t *elapsed = NULL;
if (main_struct != NULL && main_struct->opt != NULL && meta != NULL)
{
a_file = g_file_new_for_path(meta->name);
print_debug(_("Processing file: %s\n"), meta->name);
if (a_file != NULL)
{
stream = g_file_read(a_file, NULL, &error);
if (stream != NULL && error == NULL)
{
checksum = g_checksum_new(G_CHECKSUM_SHA256);
buffer = (guchar *) g_malloc(meta->blocksize);
a_hash = (guint8 *) g_malloc(digest_len);
read = g_input_stream_read((GInputStream *) stream, buffer, meta->blocksize, NULL, &error);
read_bytes = read_bytes + read;
while (read != 0 && error == NULL)
{
g_checksum_update(checksum, buffer, read);
g_checksum_get_digest(checksum, a_hash, &digest_len);
/* Need to save 'data', 'read' and digest hash in an hash_data_t structure */
hash_data = new_hash_data_t(buffer, read, a_hash);
hash_data_list = g_list_prepend(hash_data_list, hash_data);
g_checksum_reset(checksum);
digest_len = HASH_LEN;
if (read_bytes >= main_struct->opt->buffersize)
{
elapsed = new_clock_t();
print_debug(_("Sending data: %d bytes\n"), read_bytes);
/* 0. Save the list in order to keep hashs for meta-data */
hdl_copy = g_list_copy_deep(hash_data_list, copy_only_hash, NULL);
saved_list = g_list_concat(hdl_copy, saved_list);
/* 1. Send an array of hashs to Hash_Array.json server url */
answer = send_hash_array_to_server(main_struct->comm, hash_data_list);
/* 2. Keep only hashs that are needed (answer from the server) */
hash_data_list = send_all_data_to_server(main_struct, hash_data_list, answer);
/* 3. free memory of this list if any is left */
g_list_free_full(hash_data_list, free_hdt_struct);
hash_data_list = NULL;
read_bytes = 0;
end_clock(elapsed, "process_big_file_not_in_cache");
}
buffer = (guchar *) g_malloc(meta->blocksize);
a_hash = (guint8 *) g_malloc(digest_len);
read = g_input_stream_read((GInputStream *) stream, buffer, meta->blocksize, NULL, &error);
read_bytes = read_bytes + read;
}
if (error != NULL)
{
print_error(__FILE__, __LINE__, _("Error while reading file: %s\n"), error->message);
error = free_error(error);
g_list_free_full(hash_data_list, free_hdt_struct);
hash_data_list = NULL;
}
else
{
if (read_bytes > 0)
{
elapsed = new_clock_t();
print_debug(_("Sending data: %d bytes\n"), read_bytes);
/* 0. Save the list in order to keep hashs for meta-data */
hdl_copy = g_list_copy_deep(hash_data_list, copy_only_hash, NULL);
saved_list = g_list_concat(hdl_copy, saved_list);
/* 1. Send an array of hashs to Hash_Array.json server url */
answer = send_hash_array_to_server(main_struct->comm, hash_data_list);
/* 2. Keep only hashs that are needed (answer from the server) */
hash_data_list = send_all_data_to_server(main_struct, hash_data_list, answer);
/* 3. free memory of this list if any is left */
g_list_free_full(hash_data_list, free_hdt_struct);
hash_data_list = NULL;
read_bytes = 0;
end_clock(elapsed, "process_big_file_not_in_cache");
}
/* get the list in correct order (because we prepended the hashs to get speed when inserting hashs in the list) */
saved_list = g_list_reverse(saved_list);
}
free_variable(buffer);
free_variable(a_hash);
g_checksum_free(checksum);
g_input_stream_close((GInputStream *) stream, NULL, NULL);
free_object(stream);
}
else
{
print_error(__FILE__, __LINE__, _("Unable to open file for reading: %s\n"), error->message);
error = free_error(error);
}
meta->hash_data_list = saved_list;
answer = send_meta_data_to_server(main_struct, meta, TRUE);
if (answer != NULL)
{ /** @todo may be we should check that answer is something that tells that everything went Ok. */
/* Everything has been transmitted so we can save meta data into the local db cache */
/* This is usefull for file carving to avoid sending too much things to the server */
elapsed = new_clock_t();
db_save_meta_data(main_struct->database, meta, TRUE);
end_clock(elapsed, "db_save_meta_data");
}
a_file = free_object(a_file);
}
}
}
/**
* Calculates hashs for each block of blocksize bytes long on the file
* and returns a list of all hashs in correct order stored in a binary
* form to save space.
* @note This technique has some limits in term of memory footprint
* because one file is entirely in memory at a time. Saving huge
* files may not be possible with this, depending on the size of
* the file and the size of the memory.
* @todo Imagine a new way to checksum huge files because of limitations.
* May be with the local sqlite database ?
* @param a_file is the file from which we want the hashs.
* @param blocksize is the blocksize to be used to calculate hashs upon.
* @returns a GSList * list of hashs stored in a binary form.
*/
static GList *calculate_hash_data_list_for_file(GFile *a_file, gint64 blocksize)
{
GFileInputStream *stream = NULL;
GError *error = NULL;
GList *hash_data_list = NULL;
hash_data_t *hash_data = NULL;
gssize read = 0;
guchar *buffer = NULL;
GChecksum *checksum = NULL;
guint8 *a_hash = NULL;
gsize digest_len = HASH_LEN;
if (a_file != NULL)
{
stream = g_file_read(a_file, NULL, &error);
if (stream != NULL && error == NULL)
{
checksum = g_checksum_new(G_CHECKSUM_SHA256);
buffer = (guchar *) g_malloc(blocksize);
a_hash = (guint8 *) g_malloc(digest_len);
read = g_input_stream_read((GInputStream *) stream, buffer, blocksize, NULL, &error);
while (read != 0 && error == NULL)
{
g_checksum_update(checksum, buffer, read);
g_checksum_get_digest(checksum, a_hash, &digest_len);
/* Need to save data and read in hash_data_t structure */
hash_data = new_hash_data_t(buffer, read, a_hash);
hash_data_list = g_list_prepend(hash_data_list, hash_data);
g_checksum_reset(checksum);
digest_len = HASH_LEN;
buffer = (guchar *) g_malloc(blocksize);
a_hash = (guint8 *) g_malloc(digest_len);
read = g_input_stream_read((GInputStream *) stream, buffer, blocksize, NULL, &error);
}
if (error != NULL)
{
print_error(__FILE__, __LINE__, _("Error while reading file: %s\n"), error->message);
error = free_error(error);
g_list_free_full(hash_data_list, free_hdt_struct);
hash_data_list = NULL;
}
else
{
/* get the list in correct order (because we prepended the hashs to get speed when inserting hashs in the list) */
hash_data_list = g_list_reverse(hash_data_list);
}
free_variable(buffer);
free_variable(a_hash);
g_checksum_free(checksum);
g_input_stream_close((GInputStream *) stream, NULL, NULL);
free_object(stream);
}
else
{
print_error(__FILE__, __LINE__, _("Unable to open file for reading: %s\n"), error->message);
error = free_error(error);
}
}
return hash_data_list;
}
sc_bool sc_fs_storage_read_from_path(sc_segment **segments, sc_uint32 *segments_num)
{
if (g_file_test(repo_path, G_FILE_TEST_IS_DIR) == FALSE)
{
g_error("%s isn't a directory.", repo_path);
return SC_FALSE;
}
if (g_file_test(segments_path, G_FILE_TEST_IS_REGULAR) == FALSE)
{
g_message("There are no segments in %s", segments_path);
return SC_FALSE;
}
// open segments
{
GIOChannel * in_file = g_io_channel_new_file(segments_path, "r", null_ptr);
sc_fs_storage_segments_header header;
gsize bytes_num = 0;
sc_uint32 i = 0, header_size = 0;
GChecksum * checksum = null_ptr;
sc_segment * seg = null_ptr;
sc_bool is_valid = SC_TRUE;
sc_uint8 calculated_checksum[SC_STORAGE_SEG_CHECKSUM_SIZE];
g_assert(_checksum_get_size() == SC_STORAGE_SEG_CHECKSUM_SIZE);
if (!in_file)
{
g_critical("Can't open segments from: %s", segments_path);
return SC_FALSE;
}
if (g_io_channel_set_encoding(in_file, null_ptr, null_ptr) != G_IO_STATUS_NORMAL)
{
g_critical("Can't setup encoding: %s", segments_path);
return SC_FALSE;
}
if ((g_io_channel_read_chars(in_file, (gchar*)&header_size, sizeof(header_size), &bytes_num, null_ptr) != G_IO_STATUS_NORMAL) || (bytes_num != sizeof(header_size)))
{
g_critical("Can't read header size");
return SC_FALSE;
}
if (header_size != sizeof(header))
{
g_critical("Invalid header size %d != %d", header_size, (int)sizeof(header));
return SC_FALSE;
}
if ((g_io_channel_read_chars(in_file, (gchar*)&header, sizeof(header), &bytes_num, null_ptr) != G_IO_STATUS_NORMAL) || (bytes_num != sizeof(header)))
{
g_critical("Can't read header of segments: %s", segments_path);
return SC_FALSE;
}
*segments_num = header.segments_num;
/// TODO: Check version
checksum = g_checksum_new(_checksum_type());
g_assert(checksum);
g_checksum_reset(checksum);
// chek data
for (i = 0; i < *segments_num; ++i)
{
seg = sc_segment_new(i);
segments[i] = seg;
g_io_channel_read_chars(in_file, (gchar*)seg->elements, SC_SEG_ELEMENTS_SIZE_BYTE, &bytes_num, null_ptr);
sc_segment_loaded(seg);
if (bytes_num != SC_SEG_ELEMENTS_SIZE_BYTE)
{
g_error("Error while read data for segment: %d", i);
is_valid = SC_FALSE;
break;
}
g_checksum_update(checksum, (guchar*)seg->elements, SC_SEG_ELEMENTS_SIZE_BYTE);
}
if (is_valid == SC_TRUE)
{
// compare checksum
g_checksum_get_digest(checksum, calculated_checksum, &bytes_num);
if (bytes_num != SC_STORAGE_SEG_CHECKSUM_SIZE)
is_valid = SC_FALSE;
else
is_valid = (memcmp(calculated_checksum, header.checksum, SC_STORAGE_SEG_CHECKSUM_SIZE) == 0) ? SC_TRUE : SC_FALSE;
}
if (is_valid == SC_FALSE)
{
*segments_num = 0;
for (i = 0; i < SC_SEGMENT_MAX; ++i)
{
if (segments[i])
{
sc_segment_free(segments[i]);
segments[i] = null_ptr;
}
}
}
g_checksum_free(checksum);
g_io_channel_shutdown(in_file, FALSE, null_ptr);
if (is_valid == SC_FALSE)
return SC_FALSE;
}
g_message("Segments loaded: %u", *segments_num);
g_assert(fm_engine != null_ptr);
g_message("Check file memory state");
sc_bool r = sc_fm_clean_state(fm_engine) == SC_RESULT_OK;
if (r == SC_FALSE)
g_error("File memory wasn't check properly");
return r;
}
sc_bool sc_fs_storage_write_to_path(sc_segment **segments)
{
sc_uint32 idx = 0, header_size = 0;
const sc_segment *segment = 0;
sc_fs_storage_segments_header header;
GChecksum * checksum = null_ptr;
GIOChannel * output = null_ptr;
gchar * tmp_filename = null_ptr;
gsize bytes;
sc_bool result = SC_TRUE;
if (!g_file_test(repo_path, G_FILE_TEST_IS_DIR))
{
g_error("%s isn't a directory.", repo_path);
return SC_FALSE;
}
// create temporary file
output = _open_tmp_file(&tmp_filename);
memset(&header, 0, sizeof(sc_fs_storage_segments_header));
header.segments_num = 0;
header.timestamp = g_get_real_time();
header.version = sc_version_to_int(&SC_VERSION);
g_io_channel_set_encoding(output, null_ptr, null_ptr);
checksum = g_checksum_new(_checksum_type());
g_checksum_reset(checksum);
for (idx = 0; idx < SC_ADDR_SEG_MAX; idx++)
{
segment = segments[idx];
if (segment == null_ptr)
break; // stop save, because we allocate segment in order
g_checksum_update(checksum, (guchar*)segment->elements, SC_SEG_ELEMENTS_SIZE_BYTE);
}
header.segments_num = idx;
bytes = SC_STORAGE_SEG_CHECKSUM_SIZE;
g_checksum_get_digest(checksum, header.checksum, &bytes);
header_size = sizeof(header);
if (g_io_channel_write_chars(output, (gchar*)&header_size, sizeof(header_size), &bytes, null_ptr) != G_IO_STATUS_NORMAL || bytes != sizeof(header_size))
{
g_error("Can't write header size: %s", tmp_filename);
result = SC_FALSE;
goto clean;
}
if (g_io_channel_write_chars(output, (gchar*)&header, header_size, &bytes, null_ptr) != G_IO_STATUS_NORMAL || bytes != header_size)
{
g_error("Can't write header: %s", tmp_filename);
result = SC_FALSE;
goto clean;
}
for (idx = 0; idx < header.segments_num; ++idx)
{
segment = segments[idx];
g_assert(segment != null_ptr);
if (g_io_channel_write_chars(output, (gchar*)segment->elements, SC_SEG_ELEMENTS_SIZE_BYTE, &bytes, null_ptr) != G_IO_STATUS_NORMAL || bytes != SC_SEG_ELEMENTS_SIZE_BYTE)
{
g_error("Can't write segment %d into %s", idx, tmp_filename);
result = SC_FALSE;
goto clean;
}
}
if (result == SC_TRUE)
{
// rename main file
if (g_file_test(tmp_filename, G_FILE_TEST_IS_REGULAR))
{
g_io_channel_shutdown(output, TRUE, NULL);
output = null_ptr;
if (g_rename(tmp_filename, segments_path) != 0)
{
g_error("Can't rename %s -> %s", tmp_filename, segments_path);
result = SC_FALSE;
}
}
// save file memory
g_message("Save file memory state");
if (sc_fm_save(fm_engine) != SC_RESULT_OK)
g_critical("Error while saves file memory");
}
clean:
{
if (tmp_filename)
g_free(tmp_filename);
if (checksum)
g_checksum_free(checksum);
if (output)
g_io_channel_shutdown(output, TRUE, null_ptr);
}
return result;
}
