static void
destroy_cb (GtkObject *object,
ChangeDisplayInfo **info)
{
destroy_info (*info);
*info = NULL;
}
int rsvp_dataref_thread_close_reference(const struct dataref_source_info *iInfo, int rReference)
{
if (!message_queue_status() || !iInfo) return -1;
if (iInfo->respond) return 1;
pthread_t new_thread = (pthread_t) NULL;
copied_info *current_info = NULL;
if ( pthread_create( &new_thread, NULL, &thread_close_reference,
static_cast <copied_info*> (current_info = duplicate_info(iInfo, NULL, rReference, 0x00, 0x00, 0, 0)) ) != 0)
{
destroy_info(current_info);
return -1;
}
else return 0;
}
int rsvp_dataref_thread_transfer_data(const struct dataref_source_info *iInfo, int rReference,
uint8_t mMode, ssize_t oOffset, ssize_t sSize)
{
if (!message_queue_status() || !iInfo) return -1;
if (iInfo->respond) return 1;
pthread_t new_thread = (pthread_t) NULL;
copied_info *current_info = NULL;
if ( pthread_create( &new_thread, NULL, &thread_transfer_data,
static_cast <copied_info*> (current_info = duplicate_info(iInfo, NULL, rReference, 0x00, mMode, oOffset, sSize)) ) != 0)
{
destroy_info(current_info);
return -1;
}
else return 0;
}
int rsvp_dataref_thread_open_reference(const struct dataref_source_info *iInfo, text_info lLocation,
int rReference, uint8_t tType, uint8_t mMode)
{
if (!message_queue_status() || !iInfo) return -1;
if (iInfo->respond) return 1;
pthread_t new_thread = (pthread_t) NULL;
copied_info *current_info = NULL;
if ( pthread_create( &new_thread, NULL, &thread_open_reference,
static_cast <copied_info*> (current_info = duplicate_info(iInfo, lLocation, rReference, tType, mMode, 0, 0)) ) != 0)
{
destroy_info(current_info);
return -1;
}
else return 0;
}
static void *thread_alteration(void *cCopy)
{
if (!cCopy) return NULL;
copied_info *const current_info = static_cast <copied_info*> (cCopy);
command_event outcome = __rsvp_dataref_hook_alteration(¤t_info->info,
current_info->reference, current_info->offset, current_info->size);
if (current_info->info.respond && outcome != event_none)
{
command_handle new_response = short_response(current_info->info.respond, outcome);
if (new_response) send_command_no_status(new_response);
destroy_command(new_response);
}
destroy_info(current_info);
return NULL;
}
int
main(int argc, char *argv[])
{
size_t path_len, total_files;
off_t bytes_wasted, total_wasted;
char path_buffer[PATH_MAX_LEN], *hash_value;
struct file_entry_t *file_entry, *trie_entry;
SListIterator slist_iterator;
SetIterator set_iterator;
/* Step 0: Session data */
struct file_info_t file_info;
clear_info(&file_info);
/* Step 1: Parse arguments */
while (--argc) {
/* Being unable to record implies insufficient resources */
if (!record(argv[argc], &file_info)){
fprintf(stderr, "[FATAL] out of memory\n");
destroy_info(&file_info);
return (EXIT_FAILURE);
}
}
/* Step 2: Fully explore any directories specified */
#ifndef NDEBUG
printf("[DEBUG] Creating file list...\n");
#endif
while (slist_length(file_info.file_stack) > 0) {
/* Pick off the top of the file stack */
file_entry = (struct file_entry_t *)(slist_data(file_info.file_stack));
slist_remove_entry(&file_info.file_stack, file_info.file_stack);
assert(file_entry->type == DIRECTORY);
/* Copy the basename to a buffer */
memset(path_buffer, '\0', PATH_MAX_LEN);
path_len = strnlen(file_entry->path, PATH_MAX_LEN);
memcpy(path_buffer, file_entry->path, path_len);
/* Ignore cases that would cause overflow */
if (path_len < PATH_MAX_LEN) {
/* Append a trailing slash */
path_buffer[path_len] = '/';
/* Record all contents (may push onto file stack or one of the lists) */
DIR *directory = opendir(file_entry->path);
if (traverse(&file_info, directory, path_buffer, ++path_len)) {
fprintf(stderr, "[FATAL] out of memory\n");
destroy_info(&file_info);
return (EXIT_FAILURE);
} else if (closedir(directory)) {
fprintf(stderr, "[WARNING] '%s' (close failed)\n", file_entry->path);
}
}
/* Discard this entry */
destroy_entry(file_entry);
}
/* Step 3: Warn about any ignored files */
if (slist_length(file_info.bad_files) > 0) {
slist_iterate(&file_info.bad_files, &slist_iterator);
while (slist_iter_has_more(&slist_iterator)) {
file_entry = slist_iter_next(&slist_iterator);
fprintf(stderr, "[WARNING] '%s' ", file_entry->path);
switch (file_entry->type) {
case INVALID:
++file_info.invalid_files;
fprintf(stderr, "(invalid file)\n");
break;
case INACCESSIBLE:
++file_info.protected_files;
fprintf(stderr, "(protected file)\n");
break;
default:
++file_info.irregular_files;
fprintf(stderr, "(irregular file)\n");
break;
}
}
fprintf(stderr, "[WARNING] %lu file(s) ignored\n",
(long unsigned)(num_errors(&file_info)));
}
#ifndef NDEBUG
if (num_errors(&file_info) > 0) {
fprintf(stderr, "[FATAL] cannot parse entire file tree\n");
destroy_info(&file_info);
return (EXIT_FAILURE);
}
printf("[DEBUG] Found %lu / %lu valid files\n",
(unsigned long)(num_files(&file_info)),
(unsigned long)(file_info.total_files));
#endif
/* Step 4: Begin the filtering process */
#ifndef NDEBUG
printf("[DEBUG] Creating file table...\n");
#endif
if (slist_length(file_info.good_files) > 0) {
file_info.hash_trie = trie_new();
file_info.shash_trie = trie_new();
optimize_filter(&file_info);
/* Extract each file from the list (they should all be regular) */
slist_iterate(&file_info.good_files, &slist_iterator);
while (slist_iter_has_more(&slist_iterator)) {
file_entry = slist_iter_next(&slist_iterator);
assert(file_entry->type == REGULAR);
/* Perform a "shallow" hash of the file */
hash_value = hash_entry(file_entry, SHALLOW);
#ifndef NDEBUG
printf("[SHASH] %s\t*%s\n", file_entry->path, hash_value);
#endif
/* Check to see if we might have seen this file before */
if (bloom_filter_query(file_info.shash_filter, hash_value)) {
/* Get the full hash of the new file */
hash_value = hash_entry(file_entry, FULL);
#ifndef NDEBUG
printf("[+HASH] %s\t*%s\n", file_entry->path, hash_value);
#endif
archive(&file_info, file_entry);
/* Check to see if bloom failed us */
trie_entry = trie_lookup(file_info.shash_trie, file_entry->shash);
if (trie_entry == TRIE_NULL) {
#ifndef NDEBUG
printf("[DEBUG] '%s' (false positive)\n", file_entry->path);
#endif
trie_insert(file_info.shash_trie, file_entry->shash, file_entry);
} else {
/* Get the full hash of the old file */
hash_value = hash_entry(trie_entry, FULL);
#ifndef NDEBUG
if (hash_value) {
printf("[-HASH] %s\t*%s\n", trie_entry->path, hash_value);
}
#endif
archive(&file_info, trie_entry);
}
} else {
/* Add a record of this shash to the filter */
bloom_filter_insert(file_info.shash_filter, hash_value);
trie_insert(file_info.shash_trie, hash_value, file_entry);
}
}
persist("bloom_store", &file_info);
}
/* Step 5: Output results and cleanup before exit */
printf("[EXTRA] Found %lu sets of duplicates...\n",
(unsigned long)(slist_length(file_info.duplicates)));
slist_iterate(&file_info.duplicates, &slist_iterator);
for (total_files = total_wasted = bytes_wasted = 0;
slist_iter_has_more(&slist_iterator);
total_wasted += bytes_wasted)
{
Set *set = slist_iter_next(&slist_iterator);
int size = set_num_entries(set);
if (size < 2) { continue; }
printf("[EXTRA] %lu files (w/ same hash):\n", (unsigned long)(size));
set_iterate(set, &set_iterator);
for (bytes_wasted = 0;
set_iter_has_more(&set_iterator);
bytes_wasted += file_entry->size,
++total_files)
{
file_entry = set_iter_next(&set_iterator);
printf("\t%s (%lu bytes)\n",
file_entry->path,
(unsigned long)(file_entry->size));
}
}
printf("[EXTRA] %lu bytes in %lu files (wasted)\n",
(unsigned long)(total_wasted),
(unsigned long)(total_files));
destroy_info(&file_info);
return (EXIT_SUCCESS);
}
int main(int argc, char **argv)
{
#if 1
if(0 == isArbiterExist())
{
marbit_send_log(ERROR,"arbiter is not start!\n");
exit(1);
}
#endif
//init sys info
if(0 != init_sys_info())
{
marbit_send_log(ERROR,"Failed to init_sys_info!\n");
exit(1);
}
//parse the parameters
if(0 != parse_input_parameters(argc, argv))
{
destroy_info();
input_error();
exit(1);
}
//time_t begin_time = time(NULL);
// time_t do_process_data_stream_time= time(NULL);
// time_t do_process_data_aggregation_time= time(NULL);
// time_t do_process_sort_time = time(NULL);
// time_t print_baselink_time = time(NULL);
//time_t print_sorted_list_time = time(NULL);
struct timeval start, getFromArbiter, printLink, dataAggregation, quickSort, printSort, end;
gettimeofday( &start, NULL );
#if 1
//data_stream, check need get data stream from arbiter or configure file
if(0 == isNeedReadFromDB(DATA_STREAM_FILE_PATH, g_flush_interval))
{
//set_timer();
if(0 != do_process_from_arbiter())
{
marbit_send_log(ERROR,"Failed to get data stream from arbiter\n");
exit(1);
}
}
gettimeofday( &getFromArbiter, NULL );
#endif
if(SORTED_BASE_LINK == g_sorted_list_index)
{
print_baselink_info(DATA_STREAM_FILE_PATH);
gettimeofday( &printLink, NULL );
}
else if(SORTED_BASE_APP == g_sorted_list_index || SORTED_BASE_IP == g_sorted_list_index)
{
//data_aggregation
do_process_data_aggregation();
gettimeofday( &dataAggregation, NULL );
#if 1
MergeSort(&merglist);
gettimeofday( &quickSort, NULL );
printMergeList(merglist);
gettimeofday( &printSort, NULL );
destroyMergelist(merglist);
#else
struct list_head *head = &sorted_list_arry[SORTED_BASE_AGG].list;
//struct list_head *head = &sorted_list_arry[g_sorted_list_index].list;
struct list_head *first = head->next;
struct list_head *last = head->prev;
sorted_node_t *pstHead = list_entry(head, sorted_node_t, list);
quick_sort(head, first, last);
gettimeofday( &quickSort, NULL );
print_sorted_list(SORTED_BASE_AGG);
//print_sorted_list(g_sorted_list_index);
gettimeofday( &printSort, NULL );
#endif
}
destroy_info();
gettimeofday( &end, NULL );
if(logdebug.g_trace_enable_flag > 0)
{
marbit_send_log(INFO,"=======================================\n");
int timeuse = 0;
if(SORTED_BASE_LINK == g_sorted_list_index)
{
timeuse = 1000000 * ( end.tv_sec - start.tv_sec ) + end.tv_usec - start.tv_usec;
marbit_send_log(INFO,"all used time = %lu us\n", timeuse);
timeuse = 1000000 * ( getFromArbiter.tv_sec - start.tv_sec ) + getFromArbiter.tv_usec - start.tv_usec;
marbit_send_log(INFO,"getFromArbiter used time = %lu us\n", timeuse);
timeuse = 1000000 * ( printLink.tv_sec - getFromArbiter.tv_sec ) + printLink.tv_usec - getFromArbiter.tv_usec;
marbit_send_log(INFO,"printLink used time = %lu us\n", timeuse);
timeuse = 1000000 * ( end.tv_sec - printLink.tv_sec ) + end.tv_usec - printLink.tv_usec;
marbit_send_log(INFO,"destroy used time = %lu us\n", timeuse);
}
else
{
timeuse = 1000000 * ( end.tv_sec - start.tv_sec ) + end.tv_usec - start.tv_usec;
marbit_send_log(INFO,"all used time = %lu us\n", timeuse);
timeuse = 1000000 * ( getFromArbiter.tv_sec - start.tv_sec ) + getFromArbiter.tv_usec - start.tv_usec;
marbit_send_log(INFO,"getFromArbiter used time = %lu us\n", timeuse);
timeuse = 1000000 * ( dataAggregation.tv_sec - getFromArbiter.tv_sec ) + dataAggregation.tv_usec - getFromArbiter.tv_usec;
marbit_send_log(INFO,"dataAggregation used time = %lu us\n", timeuse);
timeuse = 1000000 * ( quickSort.tv_sec - dataAggregation.tv_sec ) + quickSort.tv_usec - dataAggregation.tv_usec;
marbit_send_log(INFO,"quickSort used time = %lu us\n", timeuse);
timeuse = 1000000 * ( printSort.tv_sec - quickSort.tv_sec ) + printSort.tv_usec - quickSort.tv_usec;
marbit_send_log(INFO,"printSort used time = %lu us\n", timeuse);
timeuse = 1000000 * ( end.tv_sec - printSort.tv_sec ) + end.tv_usec - printSort.tv_usec;
marbit_send_log(INFO,"destroy used time = %lu us\n", timeuse);
}
}
return 0;
}
