static struct bthread_info *btree_thread_ui_create(void)
{
int i;
struct bthread_info *bi = (struct bthread_info*)calloc(1, sizeof(struct bthread_info));
assert(bi);
bi->free_queue = mq_create(MAX_BUF, 1);
index_init();
for (i = 0; i < MAX_THREAD; i++) {
bi->node[i].id = i;
bi->node[i].count = 0;
bi->node[i].eof = 0;
bi->node[i].bi = bi;
bi->node[i].info_queue = mq_create(MAX_BUF, 0);
strcpy(bi->node[i].tmpfile, "/tmp/mem_XXXXXX");
mkstemp(bi->node[i].tmpfile);
bi->node[i].store = store_open_disk(bi->node[i].tmpfile, sizeof(struct user_info), 102400);
// bi->node[i].store = store_open_memory(sizeof(struct user_info), 102400);
// bi->node[i].tree = avlbtree_new_memory(bi->node[i].store);
bi->node[i].tree = sbtree_new_memory(bi->node[i].store);
pthread_mutex_init(&bi->node[i].mutex, NULL);
pthread_create(&bi->node[i].thread, NULL, (void *(*)(void*))insert_thread, bi->node + i);
}
return bi;
}
/* Read and initialize global index */
int
index_initialize(char *path)
{
FILE *fp;
WINDOW *w = NULL;
if (!index_initted) {
w = savescr();
dialog_clear_norefresh();
have_volumes = FALSE;
low_volume = high_volume = 0;
/* Got any media? */
if (!mediaVerify()) {
restorescr(w);
return DITEM_FAILURE;
}
/* Does it move when you kick it? */
if (!DEVICE_INIT(mediaDevice)) {
restorescr(w);
return DITEM_FAILURE;
}
dialog_clear_norefresh();
msgNotify("Attempting to fetch %s file from selected media.", path);
fp = DEVICE_GET(mediaDevice, path, TRUE);
if (!fp) {
msgConfirm("Unable to get packages/INDEX file from selected media.\n\n"
"This may be because the packages collection is not available\n"
"on the distribution media you've chosen, most likely an FTP site\n"
"without the packages collection mirrored. Please verify that\n"
"your media, or your path to the media, is correct and try again.");
DEVICE_SHUTDOWN(mediaDevice);
restorescr(w);
return DITEM_FAILURE;
}
dialog_clear_norefresh();
msgNotify("Located INDEX, now reading package data from it...");
index_init(&Top, &Plist);
if (index_read(fp, &Top)) {
msgConfirm("I/O or format error on packages/INDEX file.\n"
"Please verify media (or path to media) and try again.");
fclose(fp);
restorescr(w);
return DITEM_FAILURE;
}
fclose(fp);
index_sort(&Top);
index_initted = TRUE;
restorescr(w);
}
return DITEM_SUCCESS;
}
int main()
{
//char path[100];
//char file_name[100];
//char command[100];
//int command_num;
char path_[100] = "D:\\sysu\\Database Systems\\TPC\\tpch_2_14_3\\dbgen";
std::string path;
FILE *in_stream, *out_stream, *data_handle;
//assert (argc != 2);
//tchar2char (argv[1], path, 100);
page_init ();
buffer_init ();
index_init ();
path = path_;
//while (get_command (command))
//{
// command_num = parse_command (command);
// do_command(command_num);
//}
in_stream = fopen ((path + "\\lineitem.tbl").c_str(), "rb");
out_stream = fopen ((path + "\\hashindex.txt").c_str(), "wb+");
assert (in_stream != NULL && out_stream != NULL);
data_handle = data_import (in_stream, out_stream);
//printf ("%d\n", power (2, GLOBAL_DEPTH));
//system ("pause");
in_stream = fopen ((path + "\\testinput.txt").c_str(), "rb");
out_stream = fopen ((path + "\\testoutput.txt").c_str(), "wb");
run_test (data_handle, in_stream, out_stream);
//while (!feof (in_stream))
// printf ("%d\n", atoi (fgets (path_, 100, in_stream)));
system ("pause");
return 0;
}
struct usb_hcd *usb_hcd_alloc(struct usb_hcd_ops *ops, void *args) {
struct usb_hcd *hcd = pool_alloc(&usb_hcds);
if (!hcd) {
return NULL;
}
hcd->ops = ops;
index_init(&hcd->enumerator, 1, USB_HC_MAX_DEV, &hcd->idx_data);
dlist_head_init(&hcd->lnk);
usb_queue_init(&hcd->reset_queue);
if (ops->hcd_hci_alloc) {
hcd->hci_specific = ops->hcd_hci_alloc(hcd, args);
if (!hcd->hci_specific) {
pool_free(&usb_hcds, hcd);
return NULL;
}
}
return hcd;
}
int main(int argc, char *argv[])
{
int help = 0;
char *port = NULL;
int option;
while ((option = getopt(argc, argv, "hp:")) != -1) {
switch (option) {
case 'h':
help = 1;
break;
case 'p':
port = optarg;
break;
}
}
printf ("nyssrad - key-value server with a RESTful interface\n");
if (help) {
printf("Usage: \n");
printf("\t-p PORT - port where nyssrad should run (defaults to 1337)\n");
printf("\t-h - prints this help\n\n");
return -1;
}
if (!port) {
port = "1337";
}
index_init();
start_server(port);
return 0;
}
int backup_server(char *path) {
Jcr *jcr = new_write_jcr();
strcpy(jcr->backup_path, path);
if (access(jcr->backup_path, 4) != 0) {
free(jcr);
puts("This path does not exist or can not be read!");
return -1;
}
if (index_init() == FALSE) {
return -1;
}
jcr->job_id = get_next_job_id();
if (is_job_existed(jcr->job_id)) {
printf("job existed!\n");
free(jcr);
return FAILURE;
}
if (jcr->job_id == 0) {
destor_stat->simulation_level = simulation_level;
} else {
if (simulation_level <= SIMULATION_RECOVERY
&& destor_stat->simulation_level >= SIMULATION_APPEND
|| simulation_level >= SIMULATION_APPEND
&& destor_stat->simulation_level <= SIMULATION_RECOVERY) {
dprint(
"the current simulation level is not matched with the destor stat!");
return FAILURE;
}
}
jcr->job_volume = create_job_volume(jcr->job_id);
puts("==== backup begin ====");
puts("==== transfering begin ====");
struct timeval begin, end;
gettimeofday(&begin, 0);
if (backup(jcr) != 0) {
free(jcr);
return FAILURE;
}
gettimeofday(&end, 0);
index_destroy();
jcr->time = end.tv_sec - begin.tv_sec
+ (double) (end.tv_usec - begin.tv_usec) / (1000 * 1000);
puts("==== transferring end ====");
printf("job id: %d\n", jcr->job_id);
printf("backup path: %s\n", jcr->backup_path);
printf("number of files: %d\n", jcr->file_num);
//printf("number of chunks: %d\n", jcr->chunk_num);
printf("number of dup chunks: %d\n", jcr->number_of_dup_chunks);
printf("total size: %ld\n", jcr->job_size);
printf("dedup size: %ld\n", jcr->dedup_size);
printf("dedup efficiency: %.4f, %.4f\n",
jcr->job_size != 0 ?
(double) (jcr->dedup_size) / (double) (jcr->job_size) : 0,
jcr->job_size / (double) (jcr->job_size - jcr->dedup_size));
printf("elapsed time: %.3fs\n", jcr->time);
printf("throughput: %.2fMB/s\n",
(double) jcr->job_size / (1024 * 1024 * jcr->time));
printf("zero chunk count: %d\n", jcr->zero_chunk_count);
printf("zero_chunk_amount: %ld\n", jcr->zero_chunk_amount);
printf("rewritten_chunk_count: %d\n", jcr->rewritten_chunk_count);
printf("rewritten_chunk_amount: %ld\n", jcr->rewritten_chunk_amount);
printf("rewritten rate in amount: %.3f\n",
jcr->rewritten_chunk_amount / (double) jcr->job_size);
printf("rewritten rate in count: %.3f\n",
jcr->rewritten_chunk_count / (double) jcr->chunk_num);
destor_stat->data_amount += jcr->job_size;
destor_stat->consumed_capacity += jcr->job_size - jcr->dedup_size;
destor_stat->saved_capacity += jcr->dedup_size;
destor_stat->number_of_chunks += jcr->chunk_num;
destor_stat->number_of_dup_chunks += jcr->number_of_dup_chunks;
destor_stat->zero_chunk_count += jcr->zero_chunk_count;
destor_stat->zero_chunk_amount += jcr->zero_chunk_amount;
destor_stat->rewritten_chunk_count += jcr->rewritten_chunk_count;
destor_stat->rewritten_chunk_amount += jcr->rewritten_chunk_amount;
destor_stat->sparse_chunk_count += sparse_chunk_count;
destor_stat->sparse_chunk_amount += sparse_chunk_amount;
destor_stat->container_num = container_volume.container_num;
destor_stat->index_memory_overhead = index_memory_overhead;
printf("read_time : %.3fs, %.2fMB/s\n", jcr->read_time / 1000000,
jcr->job_size * 1000000 / jcr->read_time / 1024 / 1024);
printf("chunk_time : %.3fs, %.2fMB/s\n", jcr->chunk_time / 1000000,
jcr->job_size * 1000000 / jcr->chunk_time / 1024 / 1024);
printf("name_time : %.3fs, %.2fMB/s\n", jcr->name_time / 1000000,
jcr->job_size * 1000000 / jcr->name_time / 1024 / 1024);
printf("filter_time : %.3fs, %.2fMB/s\n", jcr->filter_time / 1000000,
jcr->job_size * 1000000 / jcr->filter_time / 1024 / 1024);
printf("write_time : %.3fs, %.2fMB/s\n", jcr->write_time / 1000000,
jcr->job_size * 1000000 / jcr->write_time / 1024 / 1024);
printf("index_search_time : %.3fs, %.2fMB/s\n", search_time / 1000000,
jcr->job_size * 1000000 / search_time / 1024 / 1024);
printf("index_update_time : %.3fs, %.2fMB/s\n", update_time / 1000000,
jcr->job_size * 1000000 / update_time / 1024 / 1024);
puts("==== backup end ====");
jcr->job_volume->job.job_id = jcr->job_id;
jcr->job_volume->job.is_del = FALSE;
jcr->job_volume->job.file_num = jcr->file_num;
jcr->job_volume->job.chunk_num = jcr->chunk_num;
strcpy(jcr->job_volume->job.backup_path, jcr->backup_path);
update_job_volume_des(jcr->job_volume);
close_job_volume(jcr->job_volume);
jcr->job_volume = 0;
double seek_time = 0.005; //5ms
double bandwidth = 120 * 1024 * 1024; //120MB/s
double index_lookup_throughput = jcr->job_size
/ (index_read_times * seek_time
+ index_read_entry_counter * 24 / bandwidth) / 1024 / 1024;
double write_data_throughput = 1.0 * jcr->job_size * bandwidth
/ (jcr->job_size - jcr->dedup_size) / 1024 / 1024;
double index_read_throughput = 1.0 * jcr->job_size / 1024 / 1024
/ (index_read_times * seek_time
+ index_read_entry_counter * 24 / bandwidth);
double index_write_throughput = 1.0 * jcr->job_size / 1024 / 1024
/ (index_write_times * seek_time
+ index_write_entry_counter * 24 / bandwidth);
double estimated_throughput = write_data_throughput;
if (estimated_throughput > index_read_throughput)
estimated_throughput = index_read_throughput;
/*if (estimated_throughput > index_write_throughput)
estimated_throughput = index_write_throughput;*/
char logfile[] = "backup.log";
int fd = open(logfile, O_WRONLY | O_CREAT, S_IRWXU);
lseek(fd, 0, SEEK_END);
char buf[512];
/*
* job id,
* chunk number,
* accumulative consumed capacity,
* deduplication ratio,
* rewritten ratio,
* total container number,
* sparse container number,
* inherited container number,
* throughput,
* index memory overhead,
* index lookups,
* index updates,
*/
sprintf(buf, "%d %d %ld %.4f %.4f %d %d %d %.2f %ld %ld %ld\n", jcr->job_id,
jcr->chunk_num, destor_stat->consumed_capacity,
jcr->job_size != 0 ?
(double) (jcr->dedup_size) / (double) (jcr->job_size) : 0,
jcr->job_size != 0 ?
(double) (jcr->rewritten_chunk_amount)
/ (double) (jcr->job_size) :
0, jcr->total_container_num, jcr->sparse_container_num,
jcr->inherited_sparse_num,
(double) jcr->job_size / (1024 * 1024 * jcr->time),
index_memory_overhead, index_read_times, index_write_times);
if (write(fd, buf, strlen(buf)) != strlen(buf)) {
}
close(fd);
free_jcr(jcr);
return SUCCESS;
}
void index_split( int *index_major, int *index_minor, int *index_dist, int dim_major, int dim_minor, int n_dom, Tindex_ptr split, int normalise )
{
int i, j, pos=0, n_el, end;
int *dims, **splits;
Tindex_ptr index;
// allocate memory
dims = (int *)calloc( n_dom, sizeof(int) );
splits = (int **)malloc( sizeof(int*)*dim_minor );
for( i=0; i<dim_minor; i++ )
splits[i] = (int *)malloc( sizeof(int)*(n_dom+1) );
// loop through the minor axis, recording where the splits are
for( j=0; j<dim_minor; j++ )
{
pos = index_minor[j];
end = index_minor[j+1];
splits[j][0] = pos;
// determine where the domain splits lie on This row/column lie
for( i=0; i<n_dom; i++ )
{
while( pos<end && index_major[pos]<index_dist[i+1] )
pos++;
splits[j][i+1] = pos;
dims[i] += (pos-splits[j][i]);
}
}
// a little bit of code to check that the caller isn't a goose
ASSERT_MSG( pos==dim_major, "index_split() : the indices you passed are pear-shaped." );
// allocate memory for the split indices
for( i=0; i<n_dom; i++ )
{
index_init( split + i, dims[i], dim_minor );
}
// split the indices, looping over the domains
for( i=0; i<n_dom; i++ )
{
// point to the index for domain i
index = split + i;
// create index list for This domain
if( dims[i] )
{
pos = 0;
for( j=0; j<dim_minor; j++ )
{
n_el = splits[j][i+1] - splits[j][i];
if( n_el )
{
memcpy( index->index_major + pos, index_major + splits[j][i], n_el*sizeof(int) );
}
pos += n_el;
index->index_minor[j+1] = pos;
}
}
}
// normalise all of the split indices to be local if needed
if( normalise )
{
// do it one domain at a time
for( i=0; i<n_dom; i++ )
{
if( dims[i] )
{
index = split + i;
j = index_dist[i];
for( pos=0; pos<dims[i]; pos++ )
{
index->index_major[pos] -= j;
}
}
}
}
// free memory
free( dims );
for( i=0; i<dim_minor; i++ )
free( splits[i] );
free( splits );
}
err_status_t
roc_test(int num_trials) {
xtd_seq_num_t local, est, ref;
ut_connection utc;
int i, num_bad_est = 0;
int delta;
uint32_t ircvd;
double failure_rate;
index_init(&local);
index_init(&ref);
index_init(&est);
printf("\n\ttesting sequential insertion...");
for (i=0; i < 2048; i++) {
delta = index_guess(&local, &est, (uint16_t) ref);
#if ROC_VERBOSE
printf("%lld, %lld, %d\n", ref, est, i);
#endif
if (ref != est) {
#if ROC_VERBOSE
printf(" *bad estimate*\n");
#endif
++num_bad_est;
}
index_advance(&ref, 1);
}
failure_rate = (double) num_bad_est / num_trials;
if (failure_rate > 0.01) {
printf("error: failure rate too high (%d bad estimates in %d trials)\n",
num_bad_est, num_trials);
return err_status_algo_fail;
}
printf("done\n");
printf("\ttesting non-sequential insertion...");
index_init(&local);
index_init(&ref);
index_init(&est);
ut_init(&utc);
for (i=0; i < num_trials; i++) {
/* get next seq num from unreliable transport simulator */
ircvd = ut_next_index(&utc);
/* set ref to value of ircvd */
ref = ircvd;
/* estimate index based on low bits of ircvd */
delta = index_guess(&local, &est, (uint16_t) ref);
#if ROC_VERBOSE
printf("ref: %lld, local: %lld, est: %lld, ircvd: %d, delta: %d\n",
ref, local, est, ircvd, delta);
#endif
/* now update local xtd_seq_num_t as necessary */
if (delta > 0)
index_advance(&local, delta);
if (ref != est) {
#if ROC_VERBOSE
printf(" *bad estimate*\n");
#endif
/* record failure event */
++num_bad_est;
/* reset local value to correct value */
local = ref;
}
}
failure_rate = (double) num_bad_est / num_trials;
if (failure_rate > 0.01) {
printf("error: failure rate too high (%d bad estimates in %d trials)\n",
num_bad_est, num_trials);
return err_status_algo_fail;
}
printf("done\n");
return err_status_ok;
}
