const char *
zfile_digest (zfile_t *self)
{
assert (self);
if (!self->digest) {
if (zfile_input (self) == -1)
return NULL; // Problem reading file
// Now calculate hash for file data, chunk by chunk
size_t blocksz = 65535;
off_t offset = 0;
self->digest = zdigest_new ();
if (!self->digest)
return NULL;
zchunk_t *chunk = zfile_read (self, blocksz, offset);
while (zchunk_size (chunk)) {
zdigest_update (self->digest,
zchunk_data (chunk), zchunk_size (chunk));
zchunk_destroy (&chunk);
// off_t is defined as long (32 bit on Windows, 64 bit otherwise)
// This guards against overflow in both contexts.
if (blocksz > LONG_MAX - offset)
return NULL;
offset += (off_t) blocksz;
chunk = zfile_read (self, blocksz, offset);
}
zchunk_destroy (&chunk);
fclose (self->handle);
self->handle = 0;
}
return zdigest_string (self->digest);
}
char *
zfile_digest (zfile_t *self)
{
if (!self->digest) {
if (zfile_input (self) == -1)
return NULL; // Problem reading file
// Now calculate hash for file data, chunk by chunk
size_t blocksz = 65535;
off_t offset = 0;
self->digest = zdigest_new ();
zchunk_t *chunk = zfile_read (self, blocksz, offset);
while (zchunk_size (chunk)) {
zdigest_update (self->digest,
zchunk_data (chunk), zchunk_size (chunk));
zchunk_destroy (&chunk);
offset += blocksz;
chunk = zfile_read (self, blocksz, offset);
}
zchunk_destroy (&chunk);
zfile_close (self);
}
return zdigest_string (self->digest);
}
static void
s_check_directory (s_agent_t *self)
{
// Get latest snapshot and build a patches list for any changes
// All patches are built using a virtual path starting at "/"
zdir_t *dir = zdir_new (self->path, NULL);
zlist_t *patches = zdir_diff (self->dir, dir, "/");
// Drop old directory and replace with latest version
zdir_destroy (&self->dir);
self->dir = dir;
while (zlist_size (patches)) {
zdir_patch_t *patch = (zdir_patch_t *) zlist_pop (patches);
if (zdir_patch_op (patch) == patch_create) {
// Shout new files to DROPS group
// Stupidest possible approach: send whole file as one frame
// Truncate file at arbitrary limit of 10MB
zfile_t *file = zdir_patch_file (patch);
if (zfile_input (file) == 0) {
zchunk_t *chunk = zfile_read (file, 10 * 1024 * 1024, 0);
assert (chunk);
zmsg_t *msg = zmsg_new ();
zmsg_addstr (msg, "CREATE");
zmsg_addstr (msg, zdir_patch_vpath (patch));
zmsg_add (msg, zframe_new (zchunk_data (chunk), zchunk_size (chunk)));
zchunk_destroy (&chunk);
zyre_shout (self->zyre, "DROPS", &msg);
}
}
zdir_patch_destroy (&patch);
}
zlist_destroy (&patches);
}
zchunk_t *
get_chunk (char *path, uint64_t chunk_size, uint64_t offset)
{
printf("[ST] GET CHUNK\n");
char *path_new = malloc(strlen("./syncfolder/") + strlen(path) + 1);
path_new[0] = '\0';
strcat(path_new, "./syncfolder/");
strcat(path_new, path);
if (zsys_file_exists (path_new)) {
printf("[ST] File exist\n");
zfile_t *file = zfile_new (".", path_new);
if (zfile_is_readable (file)) {
printf("[ST] File read\n");
zfile_input (file);
if (zfile_size (path_new) > offset) {
zchunk_t *chunk = zfile_read (file, chunk_size, offset);
zfile_destroy (&file);
return chunk;
}
else {
return NULL;
}
}
} else {
printf("[ST] File %s not exist\n", path_new);
}
return NULL;
}
static zsync_node_t *
zsync_node_new ()
{
int rc;
zsync_node_t *self = (zsync_node_t *) zmalloc (sizeof (zsync_node_t));
self->ctx = zctx_new ();
assert (self->ctx);
self->zyre = zyre_new (self->ctx);
assert (self->zyre);
// Obtain permanent UUID
self->own_uuid = zuuid_new ();
if (zsys_file_exists (UUID_FILE)) {
// Read uuid from file
zfile_t *uuid_file = zfile_new (".", UUID_FILE);
int rc = zfile_input (uuid_file); // open file for reading
assert (rc == 0);
zchunk_t *uuid_chunk = zfile_read (uuid_file, 16, 0);
assert (zchunk_size (uuid_chunk) == 16); // make sure read succeeded
zuuid_set (self->own_uuid, zchunk_data (uuid_chunk));
zfile_destroy (&uuid_file);
} else {
// Write uuid to file
zfile_t *uuid_file = zfile_new (".", UUID_FILE);
rc = zfile_output (uuid_file); // open file for writing
assert (rc == 0);
zchunk_t *uuid_bin = zchunk_new ( zuuid_data (self->own_uuid), 16);
rc = zfile_write (uuid_file, uuid_bin, 0);
assert (rc == 0);
zfile_destroy (&uuid_file);
}
// Obtain peers and states
self->peers = zlist_new ();
if (zsys_file_exists (PEER_STATES_FILE)) {
zhash_t *peer_states = zhash_new ();
int rc = zhash_load (peer_states, PEER_STATES_FILE);
assert (rc == 0);
zlist_t *uuids = zhash_keys (peer_states);
char *uuid = zlist_first (uuids);
while (uuid) {
char * state_str = zhash_lookup (peer_states, uuid);
uint64_t state;
sscanf (state_str, "%"SCNd64, &state);
zlist_append (self->peers, zsync_peer_new (uuid, state));
uuid = zlist_next (uuids);
}
}
self->zyre_peers = zhash_new ();
self->terminated = false;
return self;
}
int
zfile_test (bool verbose)
{
printf (" * zfile: ");
// @selftest
zfile_t *file = zfile_new (".", "bilbo");
assert (streq (zfile_filename (file, "."), "bilbo"));
assert (zfile_is_readable (file) == false);
zfile_destroy (&file);
// Create a test file in some random subdirectory
file = zfile_new ("./this/is/a/test", "bilbo");
int rc = zfile_output (file);
assert (rc == 0);
zchunk_t *chunk = zchunk_new (NULL, 100);
zchunk_fill (chunk, 0, 100);
// Write 100 bytes at position 1,000,000 in the file
rc = zfile_write (file, chunk, 1000000);
assert (rc == 0);
zfile_close (file);
assert (zfile_is_readable (file));
assert (zfile_cursize (file) == 1000100);
assert (!zfile_is_stable (file));
zchunk_destroy (&chunk);
zclock_sleep (1001);
assert (zfile_is_stable (file));
// Check we can read from file
rc = zfile_input (file);
assert (rc == 0);
chunk = zfile_read (file, 1000100, 0);
assert (chunk);
assert (zchunk_size (chunk) == 1000100);
zchunk_destroy (&chunk);
// Remove file and directory
zdir_t *dir = zdir_new ("./this", NULL);
assert (zdir_cursize (dir) == 1000100);
zdir_remove (dir, true);
assert (zdir_cursize (dir) == 0);
zdir_destroy (&dir);
// Check we can no longer read from file
assert (!zfile_is_readable (file));
rc = zfile_input (file);
assert (rc == -1);
zfile_destroy (&file);
// @end
printf ("OK\n");
return 0;
}
zchunk_t *
hydra_post_fetch (hydra_post_t *self, size_t size, size_t offset)
{
assert (self);
if (self->content)
return zchunk_dup (self->content);
if (size == 0)
size = self->content_size;
zfile_t *file = zfile_new (NULL, self->location);
if (zfile_input (file) == 0) {
zchunk_t *chunk = zfile_read (file, size, offset);
zfile_destroy (&file);
return chunk;
}
return NULL;
}
zconfig_t *
zconfig_load (const char *filename)
{
// Load entire file into memory as a chunk, then process it
zconfig_t *self = NULL;
zfile_t *file = zfile_new (NULL, filename);
if (!file)
return NULL;
if (zfile_input (file) == 0) {
zchunk_t *chunk = zfile_read (file, zfile_cursize (file), 0);
if (chunk) {
self = zconfig_chunk_load (chunk);
zchunk_destroy (&chunk);
if (self)
self->file = file;
zfile_close (file);
file = NULL; // Config tree now owns file handle
}
}
zfile_destroy (&file);
return self;
}
void
zfile_test (bool verbose)
{
printf (" * zfile: ");
// @selftest
zfile_t *file = zfile_new (NULL, "bilbo");
assert (streq (zfile_filename (file, "."), "bilbo"));
assert (zfile_is_readable (file) == false);
zfile_destroy (&file);
// Create a test file in some random subdirectory
file = zfile_new ("./this/is/a/test", "bilbo");
int rc = zfile_output (file);
assert (rc == 0);
zchunk_t *chunk = zchunk_new (NULL, 100);
zchunk_fill (chunk, 0, 100);
// Write 100 bytes at position 1,000,000 in the file
rc = zfile_write (file, chunk, 1000000);
assert (rc == 0);
zchunk_destroy (&chunk);
zfile_close (file);
assert (zfile_is_readable (file));
assert (zfile_cursize (file) == 1000100);
assert (!zfile_is_stable (file));
// Now append one byte to file from outside
int handle = open ("./this/is/a/test/bilbo", O_WRONLY | O_TRUNC | O_BINARY, 0);
assert (handle >= 0);
rc = write (handle, "Hello, World\n", 13);
assert (rc == 13);
close (handle);
assert (zfile_has_changed (file));
zclock_sleep (1001);
assert (zfile_has_changed (file));
assert (!zfile_is_stable (file));
zfile_restat (file);
assert (zfile_is_stable (file));
assert (streq (zfile_digest (file), "4AB299C8AD6ED14F31923DD94F8B5F5CB89DFB54"));
// Check we can read from file
rc = zfile_input (file);
assert (rc == 0);
chunk = zfile_read (file, 1000100, 0);
assert (chunk);
assert (zchunk_size (chunk) == 13);
zchunk_destroy (&chunk);
zfile_close (file);
// Try some fun with symbolic links
zfile_t *link = zfile_new ("./this/is/a/test", "bilbo.ln");
rc = zfile_output (link);
assert (rc == 0);
fprintf (zfile_handle (link), "./this/is/a/test/bilbo\n");
zfile_destroy (&link);
link = zfile_new ("./this/is/a/test", "bilbo.ln");
rc = zfile_input (link);
assert (rc == 0);
chunk = zfile_read (link, 1000100, 0);
assert (chunk);
assert (zchunk_size (chunk) == 13);
zchunk_destroy (&chunk);
zfile_destroy (&link);
// Remove file and directory
zdir_t *dir = zdir_new ("./this", NULL);
assert (zdir_cursize (dir) == 26);
zdir_remove (dir, true);
assert (zdir_cursize (dir) == 0);
zdir_destroy (&dir);
// Check we can no longer read from file
assert (zfile_is_readable (file));
zfile_restat (file);
assert (!zfile_is_readable (file));
rc = zfile_input (file);
assert (rc == -1);
zfile_destroy (&file);
// @end
printf ("OK\n");
}
static void
get_next_patch_for_client (server_t *self, client_t *client)
{
// Get next patch for client if we're not doing one already
if (client->patch == NULL)
client->patch = (zdir_patch_t *) zlist_pop (client->patches);
if (client->patch == NULL) {
client->next_event = finished_event;
return;
}
// Get virtual path from patch
fmq_msg_set_filename (client->reply, zdir_patch_vpath (client->patch));
// We can process a delete patch right away
if (zdir_patch_op (client->patch) == patch_delete) {
fmq_msg_set_sequence (client->reply, client->sequence++);
fmq_msg_set_operation (client->reply, FMQ_MSG_FILE_DELETE);
client->next_event = send_delete_event;
// No reliability in this version, assume patch delivered safely
zdir_patch_destroy (&client->patch);
}
else
if (zdir_patch_op (client->patch) == patch_create) {
// Create patch refers to file, open that for input if needed
if (client->file == NULL) {
client->file = zfile_dup (zdir_patch_file (client->patch));
if (zfile_input (client->file)) {
// File no longer available, skip it
zdir_patch_destroy (&client->patch);
zfile_destroy (&client->file);
client->next_event = next_patch_event;
return;
}
client->offset = 0;
}
// Get next chunk for file
zchunk_t *chunk = zfile_read (client->file, CHUNK_SIZE, client->offset);
assert (chunk);
// Check if we have the credit to send chunk
if (zchunk_size (chunk) <= client->credit) {
fmq_msg_set_sequence (client->reply, client->sequence++);
fmq_msg_set_operation (client->reply, FMQ_MSG_FILE_CREATE);
fmq_msg_set_offset (client->reply, client->offset);
fmq_msg_set_chunk (client->reply, zframe_new (
zchunk_data (chunk),
zchunk_size (chunk)));
client->offset += zchunk_size (chunk);
client->credit -= zchunk_size (chunk);
client->next_event = send_chunk_event;
// Zero-sized chunk means end of file
if (zchunk_size (chunk) == 0) {
zfile_destroy (&client->file);
zdir_patch_destroy (&client->patch);
}
}
else
client->next_event = no_credit_event;
zchunk_destroy (&chunk);
}
}
///
// Read chunk from file at specified position. If this was the last chunk,
// sets the eof property. Returns a null chunk in case of error.
zchunk_t * QZfile::read (size_t bytes, off_t offset)
{
zchunk_t * rv = zfile_read (self, bytes, offset);
return rv;
}
void
zfile_test (bool verbose)
{
printf (" * zfile: ");
// @selftest
const char *SELFTEST_DIR_RW = "src/selftest-rw";
const char *testbasedir = "this";
const char *testsubdir = "is/a/test";
const char *testfile = "bilbo";
const char *testlink = "bilbo.ln";
char *basedirpath = NULL; // subdir in a test, under SELFTEST_DIR_RW
char *dirpath = NULL; // subdir in a test, under basedirpath
char *filepath = NULL; // pathname to testfile in a test, in dirpath
char *linkpath = NULL; // pathname to testlink in a test, in dirpath
basedirpath = zsys_sprintf ("%s/%s", SELFTEST_DIR_RW, testbasedir);
assert (basedirpath);
dirpath = zsys_sprintf ("%s/%s", basedirpath, testsubdir);
assert (dirpath);
filepath = zsys_sprintf ("%s/%s", dirpath, testfile);
assert (filepath);
linkpath = zsys_sprintf ("%s/%s", dirpath, testlink);
assert (linkpath);
// This subtest is specifically for NULL as current directory, so
// no SELFTEST_DIR_RW here; testfile should have no slashes inside.
// Normally tests clean up in zfile_destroy(), but if a selftest run
// dies e.g. on assert(), workspace remains dirty. Better clean it up.
if (zfile_exists (testfile) ) {
if (verbose)
zsys_debug ("zfile_test() has to remove ./%s that should not have been here", testfile);
zfile_delete (testfile);
}
zfile_t *file = zfile_new (NULL, testfile);
assert (file);
assert (streq (zfile_filename (file, "."), testfile));
assert (zfile_is_readable (file) == false);
zfile_destroy (&file);
// Create a test file in some random subdirectory
if (verbose)
zsys_debug ("zfile_test() at timestamp %" PRIi64 ": "
"Creating new zfile %s",
zclock_time(), filepath );
if (zfile_exists (filepath) ) {
if (verbose)
zsys_debug ("zfile_test() has to remove %s that should not have been here", filepath);
zfile_delete (filepath);
}
file = zfile_new (dirpath, testfile);
assert (file);
int rc = zfile_output (file);
assert (rc == 0);
zchunk_t *chunk = zchunk_new (NULL, 100);
assert (chunk);
zchunk_fill (chunk, 0, 100);
// Write 100 bytes at position 1,000,000 in the file
if (verbose)
zsys_debug ("zfile_test() at timestamp %" PRIi64 ": "
"Writing 100 bytes at position 1,000,000 in the file",
zclock_time() );
rc = zfile_write (file, chunk, 1000000);
if (verbose)
zsys_debug ("zfile_test() at timestamp %" PRIi64 ": "
"Wrote 100 bytes at position 1,000,000 in the file, result code %d",
zclock_time(), rc );
assert (rc == 0);
zchunk_destroy (&chunk);
zfile_close (file);
assert (zfile_is_readable (file));
assert (zfile_cursize (file) == 1000100);
if (verbose)
zsys_debug ("zfile_test() at timestamp %" PRIi64 ": "
"Testing if file is NOT stable (is younger than 1 sec)",
zclock_time() );
assert (!zfile_is_stable (file));
if (verbose)
zsys_debug ("zfile_test() at timestamp %" PRIi64 ": "
"Passed the lag-dependent tests",
zclock_time() );
assert (zfile_digest (file));
// Now truncate file from outside
int handle = open (filepath, O_WRONLY | O_TRUNC | O_BINARY, 0);
assert (handle >= 0);
rc = write (handle, "Hello, World\n", 13);
assert (rc == 13);
close (handle);
assert (zfile_has_changed (file));
#ifdef CZMQ_BUILD_DRAFT_API
zclock_sleep ((int)zsys_file_stable_age_msec() + 50);
#else
zclock_sleep (5050);
#endif
assert (zfile_has_changed (file));
assert (!zfile_is_stable (file));
zfile_restat (file);
assert (zfile_is_stable (file));
assert (streq (zfile_digest (file), "4AB299C8AD6ED14F31923DD94F8B5F5CB89DFB54"));
// Check we can read from file
rc = zfile_input (file);
assert (rc == 0);
chunk = zfile_read (file, 1000100, 0);
assert (chunk);
assert (zchunk_size (chunk) == 13);
zchunk_destroy (&chunk);
zfile_close (file);
// Check we can read lines from file
rc = zfile_input (file);
assert (rc == 0);
const char *line = zfile_readln (file);
assert (streq (line, "Hello, World"));
line = zfile_readln (file);
assert (line == NULL);
zfile_close (file);
// Try some fun with symbolic links
zfile_t *link = zfile_new (dirpath, testlink);
assert (link);
rc = zfile_output (link);
assert (rc == 0);
fprintf (zfile_handle (link), "%s\n", filepath);
zfile_destroy (&link);
link = zfile_new (dirpath, testlink);
assert (link);
rc = zfile_input (link);
assert (rc == 0);
chunk = zfile_read (link, 1000100, 0);
assert (chunk);
assert (zchunk_size (chunk) == 13);
zchunk_destroy (&chunk);
zfile_destroy (&link);
// Remove file and directory
zdir_t *dir = zdir_new (basedirpath, NULL);
assert (dir);
assert (zdir_cursize (dir) == 26);
zdir_remove (dir, true);
assert (zdir_cursize (dir) == 0);
zdir_destroy (&dir);
// Check we can no longer read from file
assert (zfile_is_readable (file));
zfile_restat (file);
assert (!zfile_is_readable (file));
rc = zfile_input (file);
assert (rc == -1);
zfile_destroy (&file);
// This set of tests is done, free the strings for reuse
zstr_free (&basedirpath);
zstr_free (&dirpath);
zstr_free (&filepath);
zstr_free (&linkpath);
const char *eof_checkfile = "eof_checkfile";
filepath = zsys_sprintf ("%s/%s", SELFTEST_DIR_RW, eof_checkfile);
assert (filepath);
if (zfile_exists (filepath) ) {
if (verbose)
zsys_debug ("zfile_test() has to remove %s that should not have been here", filepath);
zfile_delete (filepath);
}
zstr_free (&filepath);
file = zfile_new (SELFTEST_DIR_RW, eof_checkfile);
assert (file);
// 1. Write something first
rc = zfile_output (file);
assert (rc == 0);
chunk = zchunk_new ("123456789", 9);
assert (chunk);
rc = zfile_write (file, chunk, 0);
assert (rc == 0);
zchunk_destroy (&chunk);
zfile_close (file);
assert (zfile_cursize (file) == 9);
// 2. Read the written something
rc = zfile_input (file);
assert (rc != -1);
// try to read more bytes than there is in the file
chunk = zfile_read (file, 1000, 0);
assert (zfile_eof(file));
assert (zchunk_streq (chunk, "123456789"));
zchunk_destroy (&chunk);
// reading is ok
chunk = zfile_read (file, 5, 0);
assert (!zfile_eof(file));
assert (zchunk_streq (chunk, "12345"));
zchunk_destroy (&chunk);
// read from non zero offset until the end
chunk = zfile_read (file, 5, 5);
assert (zfile_eof(file));
assert (zchunk_streq (chunk, "6789"));
zchunk_destroy (&chunk);
zfile_remove (file);
zfile_close (file);
zfile_destroy (&file);
#ifdef CZMQ_BUILD_DRAFT_API
zfile_t *tempfile = zfile_tmp ();
assert (tempfile);
assert (zfile_filename (tempfile, NULL));
assert (zsys_file_exists (zfile_filename (tempfile, NULL)));
zchunk_t *tchunk = zchunk_new ("HELLO", 6);
assert (zfile_write (tempfile, tchunk, 0) == 0);
zchunk_destroy (&tchunk);
char *filename = strdup (zfile_filename (tempfile, NULL));
zfile_destroy (&tempfile);
assert (!zsys_file_exists (filename));
zstr_free (&filename);
#endif // CZMQ_BUILD_DRAFT_API
#if defined (__WINDOWS__)
zsys_shutdown();
#endif
// @end
printf ("OK\n");
}
///
// Read chunk from file at specified position. If this was the last chunk,
// sets the eof property. Returns a null chunk in case of error.
QmlZchunk *QmlZfile::read (size_t bytes, off_t offset) {
QmlZchunk *retQ_ = new QmlZchunk ();
retQ_->self = zfile_read (self, bytes, offset);
return retQ_;
};
