static void do_close(JCR *jcr)
{
release_device(jcr->dcr);
free_attr(attr);
free_record(rec);
free_jcr(jcr);
dev->term();
}
/*
* After receiving a connection (in dircmd.c) if it is
* from the Storage daemon, this routine is called.
*/
void *handle_stored_connection(BSOCK *sd, char *job_name)
{
JCR *jcr;
/*
* With the following bmicrosleep on, running the
* SD under the debugger fails.
*/
// bmicrosleep(0, 50000); /* wait 50 millisecs */
if (!(jcr = get_jcr_by_full_name(job_name))) {
Jmsg1(NULL, M_FATAL, 0, _("SD connect failed: Job name not found: %s\n"), job_name);
Dmsg1(3, "**** Job \"%s\" not found.\n", job_name);
sd->close();
delete sd;
return NULL;
}
Dmsg1(50, "Found Job %s\n", job_name);
if (jcr->authenticated) {
Jmsg2(jcr, M_FATAL, 0, _("Hey!!!! JobId %u Job %s already authenticated.\n"),
(uint32_t)jcr->JobId, jcr->Job);
Dmsg2(50, "Hey!!!! JobId %u Job %s already authenticated.\n",
(uint32_t)jcr->JobId, jcr->Job);
sd->close();
delete sd;
free_jcr(jcr);
return NULL;
}
jcr->store_bsock = sd;
jcr->store_bsock->set_jcr(jcr);
/*
* Authenticate the Storage daemon
*/
if (jcr->authenticated || !authenticate_storagedaemon(jcr)) {
Dmsg1(50, "Authentication failed Job %s\n", jcr->Job);
Jmsg(jcr, M_FATAL, 0, _("Unable to authenticate Storage daemon\n"));
} else {
jcr->authenticated = true;
Dmsg2(50, "OK Authentication jid=%u Job %s\n", (uint32_t)jcr->JobId, jcr->Job);
}
if (!jcr->authenticated) {
jcr->setJobStatus(JS_ErrorTerminated);
}
pthread_cond_signal(&jcr->job_start_wait); /* wake waiting job */
free_jcr(jcr);
return NULL;
}
/*
* After receiving a connection (in dircmd.c) if it is
* from the File daemon, this routine is called.
*/
void handle_filed_connection(BSOCK *fd, char *job_name, int fd_version,
int sd_version)
{
JCR *jcr;
if (!(jcr=get_jcr_by_full_name(job_name))) {
Jmsg1(NULL, M_FATAL, 0, _("FD connect failed: Job name not found: %s\n"), job_name);
Dmsg1(3, "**** Job \"%s\" not found.\n", job_name);
fd->destroy();
return;
}
Dmsg1(100, "Found Filed Job %s\n", job_name);
if (jcr->authenticated) {
Jmsg2(jcr, M_FATAL, 0, _("Hey!!!! JobId %u Job %s already authenticated.\n"),
(uint32_t)jcr->JobId, jcr->Job);
Dmsg2(050, "Hey!!!! JobId %u Job %s already authenticated.\n",
(uint32_t)jcr->JobId, jcr->Job);
fd->destroy();
free_jcr(jcr);
return;
}
jcr->file_bsock = fd;
jcr->file_bsock->set_jcr(jcr);
jcr->FDVersion = fd_version;
jcr->SDVersion = sd_version;
Dmsg2(050, "fd_version=%d sd_version=%d\n", fd_version, sd_version);
/*
* Authenticate the File daemon
*/
if (jcr->authenticated || !authenticate_filed(jcr)) {
Dmsg1(50, "Authentication failed Job %s\n", jcr->Job);
Jmsg(jcr, M_FATAL, 0, _("Unable to authenticate File daemon\n"));
} else {
jcr->authenticated = true;
Dmsg2(050, "OK Authentication jid=%u Job %s\n", (uint32_t)jcr->JobId, jcr->Job);
}
if (!jcr->authenticated) {
jcr->setJobStatus(JS_ErrorTerminated);
}
Dmsg3(050, "=== Auth OK, unblock Job %s jid=%d sd_ver=%d\n", job_name, jcr->JobId, sd_version);
if (sd_version > 0) {
jcr->sd_client = true;
}
pthread_cond_signal(&jcr->job_start_wait); /* wake waiting job */
free_jcr(jcr);
return;
}
/*
* Cancel a running job on a storage daemon. Used by the interactive cancel
* command to cancel a JobId on a Storage Daemon this can be used when the
* Director already removed the Job and thinks it finished but the Storage
* Daemon still thinks its active.
*/
bool cancel_storage_daemon_job(UAContext *ua, STORERES *store, char *JobId)
{
BSOCK *sd;
JCR *control_jcr;
control_jcr = new_control_jcr("*JobCancel*", JT_SYSTEM);
control_jcr->res.wstore = store;
if (!connect_to_storage_daemon(control_jcr, 10, me->SDConnectTimeout, true)) {
ua->error_msg(_("Failed to connect to Storage daemon.\n"));
}
Dmsg0(200, "Connected to storage daemon\n");
sd = control_jcr->store_bsock;
sd->fsend("cancel Job=%s\n", JobId);
while (sd->recv() >= 0) {
ua->send_msg("%s", sd->msg);
}
sd->signal(BNET_TERMINATE);
sd->close();
control_jcr->store_bsock = NULL;
free_jcr(control_jcr);
return true;
}
/*
* Release last jcr referenced
*/
void jcr_walk_end(JCR *jcr)
{
if (jcr) {
if (jcr->JobId > 0) {
Dmsg3(dbglvl, "Free walk_end jid=%u use_count=%d Job=%s\n",
jcr->JobId, jcr->use_count(), jcr->Job);
}
free_jcr(jcr);
}
}
static void do_extract(char *devname)
{
struct stat statp;
enable_backup_privileges(NULL, 1);
jcr = setup_jcr("bextract", devname, bsr, director, VolumeName, 1); /* acquire for read */
if (!jcr) {
exit(1);
}
dev = jcr->read_dcr->dev;
if (!dev) {
exit(1);
}
dcr = jcr->read_dcr;
/* Make sure where directory exists and that it is a directory */
if (stat(where, &statp) < 0) {
berrno be;
Emsg2(M_ERROR_TERM, 0, _("Cannot stat %s. It must exist. ERR=%s\n"),
where, be.bstrerror());
}
if (!S_ISDIR(statp.st_mode)) {
Emsg1(M_ERROR_TERM, 0, _("%s must be a directory.\n"), where);
}
free(jcr->where);
jcr->where = bstrdup(where);
attr = new_attr(jcr);
compress_buf = get_memory(compress_buf_size);
acl_data.last_fname = get_pool_memory(PM_FNAME);
xattr_data.last_fname = get_pool_memory(PM_FNAME);
read_records(dcr, record_cb, mount_next_read_volume);
/* If output file is still open, it was the last one in the
* archive since we just hit an end of file, so close the file.
*/
if (is_bopen(&bfd)) {
set_attributes(jcr, attr, &bfd);
}
free_attr(attr);
free_pool_memory(acl_data.last_fname);
free_pool_memory(xattr_data.last_fname);
clean_device(jcr->dcr);
dev->term();
free_dcr(dcr);
free_jcr(jcr);
printf(_("%u files restored.\n"), num_files);
return;
}
void cancel_storage_daemon_job(JCR *jcr)
{
if (jcr->sd_canceled) {
return; /* cancel only once */
}
UAContext *ua = new_ua_context(jcr);
JCR *control_jcr = new_control_jcr("*JobCancel*", JT_SYSTEM);
BSOCK *sd;
ua->jcr = control_jcr;
if (jcr->store_bsock) {
if (!ua->jcr->wstorage) {
if (jcr->rstorage) {
copy_wstorage(ua->jcr, jcr->rstorage, _("Job resource"));
} else {
copy_wstorage(ua->jcr, jcr->wstorage, _("Job resource"));
}
} else {
USTORE store;
if (jcr->rstorage) {
store.store = jcr->rstore;
} else {
store.store = jcr->wstore;
}
set_wstorage(ua->jcr, &store);
}
if (!connect_to_storage_daemon(ua->jcr, 10, SDConnectTimeout, 1)) {
goto bail_out;
}
Dmsg0(200, "Connected to storage daemon\n");
sd = ua->jcr->store_bsock;
sd->fsend("cancel Job=%s\n", jcr->Job);
while (sd->recv() >= 0) {
}
sd->signal(BNET_TERMINATE);
sd->close();
ua->jcr->store_bsock = NULL;
jcr->sd_canceled = true;
jcr->store_bsock->set_timed_out();
jcr->store_bsock->set_terminated();
sd_msg_thread_send_signal(jcr, TIMEOUT_SIGNAL);
jcr->my_thread_send_signal(TIMEOUT_SIGNAL);
}
bail_out:
free_jcr(control_jcr);
free_ua_context(ua);
}
/*
* Get next jcr from chain, and release current one
*/
JCR *jcr_walk_next(JCR *prev_jcr)
{
JCR *jcr;
lock_jcr_chain();
jcr = (JCR *)jcrs->next(prev_jcr);
if (jcr) {
jcr->inc_use_count();
if (jcr->JobId > 0) {
Dmsg3(dbglvl, "Inc walk_next jid=%u use_count=%d Job=%s\n",
jcr->JobId, jcr->use_count(), jcr->Job);
}
}
unlock_jcr_chain();
if (prev_jcr) {
free_jcr(prev_jcr);
}
return jcr;
}
static PyObject *job_cancel(PyObject *self, PyObject *args)
{
JobId_t JobId = 0;
JCR *jcr;
bool found = false;
if (!PyArg_ParseTuple(args, "i:cancel", &JobId)) {
Dmsg0(000, "Parse tuple error in job_write\n");
return NULL;
}
foreach_jcr(jcr) {
if (jcr->JobId == 0) {
continue;
}
if (jcr->JobId == JobId) {
found = true;
break;
}
}
/* endeach_jcr(jcr) not needed because freed below */
if (!found) {
/* ***FIXME*** raise exception */
return NULL;
}
// PyEval_ReleaseLock();
UAContext *ua = new_ua_context(jcr);
ua->batch = true;
if (!cancel_job(ua, jcr)) {
/* ***FIXME*** raise exception */
return NULL;
}
free_ua_context(ua);
free_jcr(jcr);
// PyEval_AcquireLock();
Py_INCREF(Py_None);
return Py_None;
}
/*
* Cancel a running job on a storage daemon. System invoked
* non interactive version this builds a ua context and calls
* the interactive one with the silent flag set.
*/
void cancel_storage_daemon_job(JCR *jcr)
{
UAContext *ua;
JCR *control_jcr;
if (jcr->sd_canceled) {
return; /* cancel only once */
}
ua = new_ua_context(jcr);
control_jcr = new_control_jcr("*JobCancel*", JT_SYSTEM);
ua->jcr = control_jcr;
if (jcr->store_bsock) {
if (!cancel_storage_daemon_job(ua, jcr, true)) {
goto bail_out;
}
}
bail_out:
free_jcr(control_jcr);
free_ua_context(ua);
}
static void job_monitor_destructor(watchdog_t *self)
{
JCR *control_jcr = (JCR *)self->data;
free_jcr(control_jcr);
}
int main (int argc, char *argv[])
{
int ch;
JCR *jcr;
cat_op mode;
bool no_signals = false;
bool test_config = false;
char *uid = NULL;
char *gid = NULL;
start_heap = sbrk(0);
setlocale(LC_ALL, "");
bindtextdomain("bareos", LOCALEDIR);
textdomain("bareos");
init_stack_dump();
my_name_is(argc, argv, "bareos-dir");
init_msg(NULL, NULL); /* initialize message handler */
init_reload();
daemon_start_time = time(NULL);
console_command = run_console_command;
while ((ch = getopt(argc, argv, "c:d:fg:mr:stu:v?")) != -1) {
switch (ch) {
case 'c': /* specify config file */
if (configfile != NULL) {
free(configfile);
}
configfile = bstrdup(optarg);
break;
case 'd': /* set debug level */
if (*optarg == 't') {
dbg_timestamp = true;
} else {
debug_level = atoi(optarg);
if (debug_level <= 0) {
debug_level = 1;
}
}
Dmsg1(10, "Debug level = %d\n", debug_level);
break;
case 'f': /* run in foreground */
background = false;
break;
case 'g': /* set group id */
gid = optarg;
break;
case 'm': /* print kaboom output */
prt_kaboom = true;
break;
case 'r': /* run job */
if (runjob != NULL) {
free(runjob);
}
if (optarg) {
runjob = bstrdup(optarg);
}
break;
case 's': /* turn off signals */
no_signals = true;
break;
case 't': /* test config */
test_config = true;
break;
case 'u': /* set uid */
uid = optarg;
break;
case 'v': /* verbose */
verbose++;
break;
case '?':
default:
usage();
}
}
argc -= optind;
argv += optind;
if (!no_signals) {
init_signals(terminate_dird);
}
if (argc) {
if (configfile != NULL) {
free(configfile);
}
configfile = bstrdup(*argv);
argc--;
argv++;
}
if (argc) {
usage();
}
if (configfile == NULL) {
configfile = bstrdup(CONFIG_FILE);
}
/*
* See if we want to drop privs.
*/
if (geteuid() == 0) {
drop(uid, gid, false); /* reduce privileges if requested */
}
my_config = new_config_parser();
parse_dir_config(my_config, configfile, M_ERROR_TERM);
if (init_crypto() != 0) {
Jmsg((JCR *)NULL, M_ERROR_TERM, 0, _("Cryptography library initialization failed.\n"));
goto bail_out;
}
if (!check_resources()) {
Jmsg((JCR *)NULL, M_ERROR_TERM, 0, _("Please correct configuration file: %s\n"), configfile);
goto bail_out;
}
if (!test_config) { /* we don't need to do this block in test mode */
if (background) {
daemon_start();
init_stack_dump(); /* grab new pid */
}
/* Create pid must come after we are a daemon -- so we have our final pid */
create_pid_file(me->pid_directory, "bareos-dir",
get_first_port_host_order(me->DIRaddrs));
read_state_file(me->working_directory, "bareos-dir",
get_first_port_host_order(me->DIRaddrs));
}
set_jcr_in_tsd(INVALID_JCR);
set_thread_concurrency(me->MaxConcurrentJobs * 2 +
4 /* UA */ + 5 /* sched+watchdog+jobsvr+misc */);
lmgr_init_thread(); /* initialize the lockmanager stack */
load_dir_plugins(me->plugin_directory, me->plugin_names);
/*
* If we are in testing mode, we don't try to fix the catalog
*/
mode = (test_config) ? CHECK_CONNECTION : UPDATE_AND_FIX;
if (!check_catalog(mode)) {
Jmsg((JCR *)NULL, M_ERROR_TERM, 0, _("Please correct configuration file: %s\n"), configfile);
goto bail_out;
}
if (test_config) {
terminate_dird(0);
}
if (!initialize_sql_pooling()) {
Jmsg((JCR *)NULL, M_ERROR_TERM, 0, _("Please correct configuration file: %s\n"), configfile);
goto bail_out;
}
my_name_is(0, NULL, me->name()); /* set user defined name */
cleanup_old_files();
p_db_log_insert = (db_log_insert_func)dir_db_log_insert;
#if !defined(HAVE_WIN32)
signal(SIGHUP, reload_config);
#endif
init_console_msg(working_directory);
Dmsg0(200, "Start UA server\n");
start_UA_server(me->DIRaddrs);
start_watchdog(); /* start network watchdog thread */
if (me->jcr_watchdog_time) {
init_jcr_subsystem(me->jcr_watchdog_time); /* start JCR watchdogs etc. */
}
init_job_server(me->MaxConcurrentJobs);
dbg_jcr_add_hook(db_debug_print); /* used to debug B_DB connexion after fatal signal */
// init_device_resources();
Dmsg0(200, "wait for next job\n");
/* Main loop -- call scheduler to get next job to run */
while ( (jcr = wait_for_next_job(runjob)) ) {
run_job(jcr); /* run job */
free_jcr(jcr); /* release jcr */
set_jcr_in_tsd(INVALID_JCR);
if (runjob) { /* command line, run a single job? */
break; /* yes, terminate */
}
}
terminate_dird(0);
bail_out:
return 0;
}
void *handle_stored_connection(BSOCK *sd)
{
JCR *jcr;
char job_name[MAX_NAME_LENGTH];
/*
* Do a sanity check on the message received
*/
if (sd->msglen < 25 || sd->msglen > 256) {
Dmsg1(000, "<filed: %s", sd->msg);
Emsg2(M_ERROR, 0, _("Invalid connection from %s. Len=%d\n"), sd->who(), sd->msglen);
bmicrosleep(5, 0); /* make user wait 5 seconds */
sd->close();
return NULL;
}
if (sscanf(sd->msg, "Hello Storage calling Start Job %127s", job_name) != 1) {
char addr[64];
char *who = bnet_get_peer(sd, addr, sizeof(addr)) ? sd->who() : addr;
sd->msg[100] = 0;
Dmsg2(dbglvl, "Bad Hello command from Director at %s: %s\n", sd->who(), sd->msg);
Jmsg2(NULL, M_FATAL, 0, _("Bad Hello command from Director at %s: %s\n"), who, sd->msg);
sd->close();
return NULL;
}
if (!(jcr = get_jcr_by_full_name(job_name))) {
Jmsg1(NULL, M_FATAL, 0, _("SD connect failed: Job name not found: %s\n"), job_name);
Dmsg1(3, "**** Job \"%s\" not found.\n", job_name);
sd->close();
return NULL;
}
Dmsg1(50, "Found Job %s\n", job_name);
jcr->store_bsock = sd;
jcr->store_bsock->set_jcr(jcr);
/*
* Authenticate the Storage Daemon.
*/
if (!authenticate_storagedaemon(jcr)) {
Dmsg1(50, "Authentication failed Job %s\n", jcr->Job);
Jmsg(jcr, M_FATAL, 0, _("Unable to authenticate File daemon\n"));
jcr->setJobStatus(JS_ErrorTerminated);
} else {
Dmsg2(50, "OK Authentication jid=%u Job %s\n", (uint32_t)jcr->JobId, jcr->Job);
}
if (!jcr->max_bandwidth) {
if (jcr->director->max_bandwidth_per_job) {
jcr->max_bandwidth = jcr->director->max_bandwidth_per_job;
} else if (me->max_bandwidth_per_job) {
jcr->max_bandwidth = me->max_bandwidth_per_job;
}
}
sd->set_bwlimit(jcr->max_bandwidth);
if (me->allow_bw_bursting) {
sd->set_bwlimit_bursting();
}
free_jcr(jcr);
return NULL;
}
int main (int argc, char *argv[])
{
int ch;
char *iVolumeName = NULL;
char *oVolumeName = NULL;
bool ignore_label_errors = false;
bool ok;
setlocale(LC_ALL, "");
bindtextdomain("bacula", LOCALEDIR);
textdomain("bacula");
init_stack_dump();
my_name_is(argc, argv, "bcopy");
init_msg(NULL, NULL);
while ((ch = getopt(argc, argv, "b:c:d:i:o:pvw:?")) != -1) {
switch (ch) {
case 'b':
bsr = parse_bsr(NULL, optarg);
break;
case 'c': /* specify config file */
if (configfile != NULL) {
free(configfile);
}
configfile = bstrdup(optarg);
break;
case 'd': /* debug level */
if (*optarg == 't') {
dbg_timestamp = true;
} else {
debug_level = atoi(optarg);
if (debug_level <= 0) {
debug_level = 1;
}
}
break;
case 'i': /* input Volume name */
iVolumeName = optarg;
break;
case 'o': /* output Volume name */
oVolumeName = optarg;
break;
case 'p':
ignore_label_errors = true;
forge_on = true;
break;
case 'v':
verbose++;
break;
case 'w':
wd = optarg;
break;
case '?':
default:
usage();
}
}
argc -= optind;
argv += optind;
if (argc != 2) {
Pmsg0(0, _("Wrong number of arguments: \n"));
usage();
}
OSDependentInit();
working_directory = wd;
if (configfile == NULL) {
configfile = bstrdup(CONFIG_FILE);
}
config = new_config_parser();
parse_sd_config(config, configfile, M_ERROR_TERM);
/* Setup and acquire input device for reading */
Dmsg0(100, "About to setup input jcr\n");
in_jcr = setup_jcr("bcopy", argv[0], bsr, iVolumeName, 1); /* read device */
if (!in_jcr) {
exit(1);
}
in_jcr->ignore_label_errors = ignore_label_errors;
in_dev = in_jcr->dcr->dev;
if (!in_dev) {
exit(1);
}
/* Setup output device for writing */
Dmsg0(100, "About to setup output jcr\n");
out_jcr = setup_jcr("bcopy", argv[1], bsr, oVolumeName, 0); /* no acquire */
if (!out_jcr) {
exit(1);
}
out_dev = out_jcr->dcr->dev;
if (!out_dev) {
exit(1);
}
Dmsg0(100, "About to acquire device for writing\n");
/* For we must now acquire the device for writing */
out_dev->r_dlock();
if (out_dev->open(out_jcr->dcr, OPEN_READ_WRITE) < 0) {
Emsg1(M_FATAL, 0, _("dev open failed: %s\n"), out_dev->errmsg);
out_dev->dunlock();
exit(1);
}
out_dev->dunlock();
if (!acquire_device_for_append(out_jcr->dcr)) {
free_jcr(in_jcr);
exit(1);
}
out_block = out_jcr->dcr->block;
ok = read_records(in_jcr->dcr, record_cb, mount_next_read_volume);
if (ok || out_dev->can_write()) {
if (!write_block_to_device(out_jcr->dcr)) {
Pmsg0(000, _("Write of last block failed.\n"));
}
}
Pmsg2(000, _("%u Jobs copied. %u records copied.\n"), jobs, records);
free_jcr(in_jcr);
free_jcr(out_jcr);
in_dev->term();
out_dev->term();
return 0;
}
int backup_server(char *path) {
Jcr *jcr = new_write_jcr();
strcpy(jcr->backup_path, path);
if (access(jcr->backup_path, 4) != 0) {
free(jcr);
puts("This path does not exist or can not be read!");
return -1;
}
if (index_init() == FALSE) {
return -1;
}
jcr->job_id = get_next_job_id();
if (is_job_existed(jcr->job_id)) {
printf("job existed!\n");
free(jcr);
return FAILURE;
}
if (jcr->job_id == 0) {
destor_stat->simulation_level = simulation_level;
} else {
if (simulation_level <= SIMULATION_RECOVERY
&& destor_stat->simulation_level >= SIMULATION_APPEND
|| simulation_level >= SIMULATION_APPEND
&& destor_stat->simulation_level <= SIMULATION_RECOVERY) {
dprint(
"the current simulation level is not matched with the destor stat!");
return FAILURE;
}
}
jcr->job_volume = create_job_volume(jcr->job_id);
puts("==== backup begin ====");
puts("==== transfering begin ====");
struct timeval begin, end;
gettimeofday(&begin, 0);
if (backup(jcr) != 0) {
free(jcr);
return FAILURE;
}
gettimeofday(&end, 0);
index_destroy();
jcr->time = end.tv_sec - begin.tv_sec
+ (double) (end.tv_usec - begin.tv_usec) / (1000 * 1000);
puts("==== transferring end ====");
printf("job id: %d\n", jcr->job_id);
printf("backup path: %s\n", jcr->backup_path);
printf("number of files: %d\n", jcr->file_num);
//printf("number of chunks: %d\n", jcr->chunk_num);
printf("number of dup chunks: %d\n", jcr->number_of_dup_chunks);
printf("total size: %ld\n", jcr->job_size);
printf("dedup size: %ld\n", jcr->dedup_size);
printf("dedup efficiency: %.4f, %.4f\n",
jcr->job_size != 0 ?
(double) (jcr->dedup_size) / (double) (jcr->job_size) : 0,
jcr->job_size / (double) (jcr->job_size - jcr->dedup_size));
printf("elapsed time: %.3fs\n", jcr->time);
printf("throughput: %.2fMB/s\n",
(double) jcr->job_size / (1024 * 1024 * jcr->time));
printf("zero chunk count: %d\n", jcr->zero_chunk_count);
printf("zero_chunk_amount: %ld\n", jcr->zero_chunk_amount);
printf("rewritten_chunk_count: %d\n", jcr->rewritten_chunk_count);
printf("rewritten_chunk_amount: %ld\n", jcr->rewritten_chunk_amount);
printf("rewritten rate in amount: %.3f\n",
jcr->rewritten_chunk_amount / (double) jcr->job_size);
printf("rewritten rate in count: %.3f\n",
jcr->rewritten_chunk_count / (double) jcr->chunk_num);
destor_stat->data_amount += jcr->job_size;
destor_stat->consumed_capacity += jcr->job_size - jcr->dedup_size;
destor_stat->saved_capacity += jcr->dedup_size;
destor_stat->number_of_chunks += jcr->chunk_num;
destor_stat->number_of_dup_chunks += jcr->number_of_dup_chunks;
destor_stat->zero_chunk_count += jcr->zero_chunk_count;
destor_stat->zero_chunk_amount += jcr->zero_chunk_amount;
destor_stat->rewritten_chunk_count += jcr->rewritten_chunk_count;
destor_stat->rewritten_chunk_amount += jcr->rewritten_chunk_amount;
destor_stat->sparse_chunk_count += sparse_chunk_count;
destor_stat->sparse_chunk_amount += sparse_chunk_amount;
destor_stat->container_num = container_volume.container_num;
destor_stat->index_memory_overhead = index_memory_overhead;
printf("read_time : %.3fs, %.2fMB/s\n", jcr->read_time / 1000000,
jcr->job_size * 1000000 / jcr->read_time / 1024 / 1024);
printf("chunk_time : %.3fs, %.2fMB/s\n", jcr->chunk_time / 1000000,
jcr->job_size * 1000000 / jcr->chunk_time / 1024 / 1024);
printf("name_time : %.3fs, %.2fMB/s\n", jcr->name_time / 1000000,
jcr->job_size * 1000000 / jcr->name_time / 1024 / 1024);
printf("filter_time : %.3fs, %.2fMB/s\n", jcr->filter_time / 1000000,
jcr->job_size * 1000000 / jcr->filter_time / 1024 / 1024);
printf("write_time : %.3fs, %.2fMB/s\n", jcr->write_time / 1000000,
jcr->job_size * 1000000 / jcr->write_time / 1024 / 1024);
printf("index_search_time : %.3fs, %.2fMB/s\n", search_time / 1000000,
jcr->job_size * 1000000 / search_time / 1024 / 1024);
printf("index_update_time : %.3fs, %.2fMB/s\n", update_time / 1000000,
jcr->job_size * 1000000 / update_time / 1024 / 1024);
puts("==== backup end ====");
jcr->job_volume->job.job_id = jcr->job_id;
jcr->job_volume->job.is_del = FALSE;
jcr->job_volume->job.file_num = jcr->file_num;
jcr->job_volume->job.chunk_num = jcr->chunk_num;
strcpy(jcr->job_volume->job.backup_path, jcr->backup_path);
update_job_volume_des(jcr->job_volume);
close_job_volume(jcr->job_volume);
jcr->job_volume = 0;
double seek_time = 0.005; //5ms
double bandwidth = 120 * 1024 * 1024; //120MB/s
double index_lookup_throughput = jcr->job_size
/ (index_read_times * seek_time
+ index_read_entry_counter * 24 / bandwidth) / 1024 / 1024;
double write_data_throughput = 1.0 * jcr->job_size * bandwidth
/ (jcr->job_size - jcr->dedup_size) / 1024 / 1024;
double index_read_throughput = 1.0 * jcr->job_size / 1024 / 1024
/ (index_read_times * seek_time
+ index_read_entry_counter * 24 / bandwidth);
double index_write_throughput = 1.0 * jcr->job_size / 1024 / 1024
/ (index_write_times * seek_time
+ index_write_entry_counter * 24 / bandwidth);
double estimated_throughput = write_data_throughput;
if (estimated_throughput > index_read_throughput)
estimated_throughput = index_read_throughput;
/*if (estimated_throughput > index_write_throughput)
estimated_throughput = index_write_throughput;*/
char logfile[] = "backup.log";
int fd = open(logfile, O_WRONLY | O_CREAT, S_IRWXU);
lseek(fd, 0, SEEK_END);
char buf[512];
/*
* job id,
* chunk number,
* accumulative consumed capacity,
* deduplication ratio,
* rewritten ratio,
* total container number,
* sparse container number,
* inherited container number,
* throughput,
* index memory overhead,
* index lookups,
* index updates,
*/
sprintf(buf, "%d %d %ld %.4f %.4f %d %d %d %.2f %ld %ld %ld\n", jcr->job_id,
jcr->chunk_num, destor_stat->consumed_capacity,
jcr->job_size != 0 ?
(double) (jcr->dedup_size) / (double) (jcr->job_size) : 0,
jcr->job_size != 0 ?
(double) (jcr->rewritten_chunk_amount)
/ (double) (jcr->job_size) :
0, jcr->total_container_num, jcr->sparse_container_num,
jcr->inherited_sparse_num,
(double) jcr->job_size / (1024 * 1024 * jcr->time),
index_memory_overhead, index_read_times, index_write_times);
if (write(fd, buf, strlen(buf)) != strlen(buf)) {
}
close(fd);
free_jcr(jcr);
return SUCCESS;
}
/*
* Director requests us to start a job
* Basic tasks done here:
* - We pickup the JobId to be run from the Director.
* - We pickup the device, media, and pool from the Director
* - Wait for a connection from the File Daemon (FD)
* - Accept commands from the FD (i.e. run the job)
* - Return when the connection is terminated or
* there is an error.
*/
bool job_cmd(JCR *jcr)
{
int32_t JobId;
char sd_auth_key[200];
char spool_size[30];
char seed[100];
BSOCK *dir = jcr->dir_bsock;
POOL_MEM job_name, client_name, job, fileset_name, fileset_md5;
int32_t JobType, level, spool_attributes, no_attributes, spool_data;
int32_t write_part_after_job, PreferMountedVols;
int32_t rerunning;
int32_t is_client;
int stat;
JCR *ojcr;
/*
* Get JobId and permissions from Director
*/
Dmsg1(100, "<dird: %s", dir->msg);
bstrncpy(spool_size, "0", sizeof(spool_size));
stat = sscanf(dir->msg, jobcmd, &JobId, job.c_str(), job_name.c_str(),
client_name.c_str(),
&JobType, &level, fileset_name.c_str(), &no_attributes,
&spool_attributes, fileset_md5.c_str(), &spool_data,
&write_part_after_job, &PreferMountedVols, spool_size,
&rerunning, &jcr->VolSessionId, &jcr->VolSessionTime,
&is_client, &sd_auth_key);
if (stat != 19) {
pm_strcpy(jcr->errmsg, dir->msg);
dir->fsend(BAD_job, stat, jcr->errmsg);
Dmsg1(100, ">dird: %s", dir->msg);
jcr->setJobStatus(JS_ErrorTerminated);
return false;
}
jcr->rerunning = rerunning;
jcr->sd_client = is_client;
if (is_client) {
jcr->sd_auth_key = bstrdup(sd_auth_key);
}
Dmsg3(100, "rerunning=%d VolSesId=%d VolSesTime=%d\n", jcr->rerunning,
jcr->VolSessionId, jcr->VolSessionTime);
/*
* Since this job could be rescheduled, we
* check to see if we have it already. If so
* free the old jcr and use the new one.
*/
ojcr = get_jcr_by_full_name(job.c_str());
if (ojcr && !ojcr->authenticated) {
Dmsg2(100, "Found ojcr=0x%x Job %s\n", (unsigned)(intptr_t)ojcr, job.c_str());
free_jcr(ojcr);
}
jcr->JobId = JobId;
Dmsg2(800, "Start JobId=%d %p\n", JobId, jcr);
set_jcr_in_tsd(jcr);
/*
* If job rescheduled because previous was incomplete,
* the Resched flag is set and VolSessionId and VolSessionTime
* are given to us (same as restarted job).
*/
if (!jcr->rerunning) {
jcr->VolSessionId = newVolSessionId();
jcr->VolSessionTime = VolSessionTime;
}
bstrncpy(jcr->Job, job, sizeof(jcr->Job));
unbash_spaces(job_name);
jcr->job_name = get_pool_memory(PM_NAME);
pm_strcpy(jcr->job_name, job_name);
unbash_spaces(client_name);
jcr->client_name = get_pool_memory(PM_NAME);
pm_strcpy(jcr->client_name, client_name);
unbash_spaces(fileset_name);
jcr->fileset_name = get_pool_memory(PM_NAME);
pm_strcpy(jcr->fileset_name, fileset_name);
jcr->setJobType(JobType);
jcr->setJobLevel(level);
jcr->no_attributes = no_attributes;
jcr->spool_attributes = spool_attributes;
jcr->spool_data = spool_data;
jcr->spool_size = str_to_int64(spool_size);
jcr->write_part_after_job = write_part_after_job;
jcr->fileset_md5 = get_pool_memory(PM_NAME);
pm_strcpy(jcr->fileset_md5, fileset_md5);
jcr->PreferMountedVols = PreferMountedVols;
jcr->authenticated = false;
/*
* Pass back an authorization key for the File daemon
*/
if (jcr->sd_client) {
bstrncpy(sd_auth_key, "xxx", 3);
} else {
bsnprintf(seed, sizeof(seed), "%p%d", jcr, JobId);
make_session_key(sd_auth_key, seed, 1);
}
dir->fsend(OKjob, jcr->VolSessionId, jcr->VolSessionTime, sd_auth_key);
Dmsg2(150, ">dird jid=%u: %s", (uint32_t)jcr->JobId, dir->msg);
/* If not client, set key, otherwise it is already set */
if (!jcr->sd_client) {
jcr->sd_auth_key = bstrdup(sd_auth_key);
memset(sd_auth_key, 0, sizeof(sd_auth_key));
}
new_plugins(jcr); /* instantiate the plugins */
generate_daemon_event(jcr, "JobStart");
generate_plugin_event(jcr, bsdEventJobStart, (void *)"JobStart");
return true;
}
int main (int argc, char *argv[])
{
int ch;
bool ok;
char *iVolumeName = NULL;
char *oVolumeName = NULL;
char *DirectorName = NULL;
DIRRES *director = NULL;
bool ignore_label_errors = false;
DCR *in_dcr, *out_dcr;
setlocale(LC_ALL, "");
bindtextdomain("bareos", LOCALEDIR);
textdomain("bareos");
init_stack_dump();
my_name_is(argc, argv, "bcopy");
lmgr_init_thread();
init_msg(NULL, NULL);
while ((ch = getopt(argc, argv, "b:c:D:d:i:o:pvw:?")) != -1) {
switch (ch) {
case 'b':
bsr = parse_bsr(NULL, optarg);
break;
case 'c': /* specify config file */
if (configfile != NULL) {
free(configfile);
}
configfile = bstrdup(optarg);
break;
case 'D': /* specify director name */
if (DirectorName != NULL) {
free(DirectorName);
}
DirectorName = bstrdup(optarg);
break;
case 'd': /* debug level */
if (*optarg == 't') {
dbg_timestamp = true;
} else {
debug_level = atoi(optarg);
if (debug_level <= 0) {
debug_level = 1;
}
}
break;
case 'i': /* input Volume name */
iVolumeName = optarg;
break;
case 'o': /* output Volume name */
oVolumeName = optarg;
break;
case 'p':
ignore_label_errors = true;
forge_on = true;
break;
case 'v':
verbose++;
break;
case 'w':
wd = optarg;
break;
case '?':
default:
usage();
}
}
argc -= optind;
argv += optind;
if (argc != 2) {
Pmsg0(0, _("Wrong number of arguments: \n"));
usage();
}
OSDependentInit();
working_directory = wd;
if (configfile == NULL) {
configfile = bstrdup(CONFIG_FILE);
}
my_config = new_config_parser();
parse_sd_config(my_config, configfile, M_ERROR_TERM);
if (DirectorName) {
foreach_res(director, R_DIRECTOR) {
if (bstrcmp(director->hdr.name, DirectorName)) {
break;
}
}
if (!director) {
Emsg2(M_ERROR_TERM, 0, _("No Director resource named %s defined in %s. Cannot continue.\n"),
DirectorName, configfile);
}
}
load_sd_plugins(me->plugin_directory, me->plugin_names);
read_crypto_cache(me->working_directory, "bareos-sd",
get_first_port_host_order(me->SDaddrs));
/*
* Setup and acquire input device for reading
*/
Dmsg0(100, "About to setup input jcr\n");
in_dcr = New(DCR);
in_jcr = setup_jcr("bcopy", argv[0], bsr, director, in_dcr, iVolumeName, true); /* read device */
if (!in_jcr) {
exit(1);
}
in_jcr->ignore_label_errors = ignore_label_errors;
in_dev = in_jcr->dcr->dev;
if (!in_dev) {
exit(1);
}
/*
* Setup output device for writing
*/
Dmsg0(100, "About to setup output jcr\n");
out_dcr = New(DCR);
out_jcr = setup_jcr("bcopy", argv[1], bsr, director, out_dcr, oVolumeName, false); /* write device */
if (!out_jcr) {
exit(1);
}
out_dev = out_jcr->dcr->dev;
if (!out_dev) {
exit(1);
}
Dmsg0(100, "About to acquire device for writing\n");
/*
* For we must now acquire the device for writing
*/
out_dev->rLock(false);
if (!out_dev->open(out_jcr->dcr, OPEN_READ_WRITE)) {
Emsg1(M_FATAL, 0, _("dev open failed: %s\n"), out_dev->errmsg);
out_dev->Unlock();
exit(1);
}
out_dev->Unlock();
if (!acquire_device_for_append(out_jcr->dcr)) {
free_jcr(in_jcr);
exit(1);
}
out_block = out_jcr->dcr->block;
ok = read_records(in_jcr->dcr, record_cb, mount_next_read_volume);
if (ok || out_dev->can_write()) {
if (!out_jcr->dcr->write_block_to_device()) {
Pmsg0(000, _("Write of last block failed.\n"));
}
}
Pmsg2(000, _("%u Jobs copied. %u records copied.\n"), jobs, records);
in_dev->term();
out_dev->term();
free_jcr(in_jcr);
free_jcr(out_jcr);
return 0;
}
int main(int argc, char *const *argv)
{
FF_PKT *ff;
char name[1000];
bool quiet = false;
int i, ch;
char *inc = NULL;
char *exc = NULL;
FILE *fd;
setlocale(LC_ALL, "");
bindtextdomain("bacula", LOCALEDIR);
textdomain("bacula");
lmgr_init_thread();
while ((ch = getopt(argc, argv, "ad:e:i:q?")) != -1) {
switch (ch) {
case 'a': /* print extended attributes *debug* */
attrs = 1;
break;
case 'd': /* set debug level */
if (*optarg == 't') {
dbg_timestamp = true;
} else {
debug_level = atoi(optarg);
if (debug_level <= 0) {
debug_level = 1;
}
}
break;
case 'e': /* exclude patterns */
exc = optarg;
break;
case 'i': /* include patterns */
inc = optarg;
break;
case 'q':
quiet = true;
break;
case '?':
default:
usage();
}
}
argc -= optind;
argv += optind;
jcr = new_jcr(sizeof(JCR), NULL);
ff = init_find_files();
if (argc == 0 && !inc) {
add_fname_to_include_list(ff, 0, "/"); /* default to / */
} else {
for (i=0; i < argc; i++) {
if (strcmp(argv[i], "-") == 0) {
while (fgets(name, sizeof(name)-1, stdin)) {
strip_trailing_junk(name);
add_fname_to_include_list(ff, 0, name);
}
continue;
}
add_fname_to_include_list(ff, 0, argv[i]);
}
}
if (inc) {
fd = fopen(inc, "rb");
if (!fd) {
printf(_("Could not open include file: %s\n"), inc);
exit(1);
}
while (fgets(name, sizeof(name)-1, fd)) {
strip_trailing_junk(name);
add_fname_to_include_list(ff, 0, name);
}
fclose(fd);
}
if (exc) {
fd = fopen(exc, "rb");
if (!fd) {
printf(_("Could not open exclude file: %s\n"), exc);
exit(1);
}
while (fgets(name, sizeof(name)-1, fd)) {
strip_trailing_junk(name);
add_fname_to_exclude_list(ff, name);
}
fclose(fd);
}
if (quiet) {
match_files(jcr, ff, count_files);
} else {
match_files(jcr, ff, print_file);
}
printf(_("Files seen = %d\n"), num_files);
term_include_exclude_files(ff);
term_find_files(ff);
free_jcr(jcr);
term_last_jobs_list(); /* free jcr chain */
close_memory_pool();
lmgr_cleanup_main();
sm_dump(false);
exit(0);
}
static bool list_nextvol(UAContext *ua, int ndays)
{
JOBRES *job;
JCR *jcr;
USTORERES store;
RUNRES *run;
utime_t runtime;
bool found = false;
MEDIA_DBR mr;
POOL_DBR pr;
int i = find_arg_with_value(ua, "job");
if (i <= 0) {
if ((job = select_job_resource(ua)) == NULL) {
return false;
}
} else {
job = (JOBRES *)GetResWithName(R_JOB, ua->argv[i]);
if (!job) {
Jmsg(ua->jcr, M_ERROR, 0, _("%s is not a job name.\n"), ua->argv[i]);
if ((job = select_job_resource(ua)) == NULL) {
return false;
}
}
}
jcr = new_jcr(sizeof(JCR), dird_free_jcr);
for (run=NULL; (run = find_next_run(run, job, runtime, ndays)); ) {
if (!complete_jcr_for_job(jcr, job, run->pool)) {
found = false;
goto get_out;
}
if (!jcr->jr.PoolId) {
ua->error_msg(_("Could not find Pool for Job %s\n"), job->name());
continue;
}
memset(&pr, 0, sizeof(pr));
pr.PoolId = jcr->jr.PoolId;
if (!db_get_pool_record(jcr, jcr->db, &pr)) {
bstrncpy(pr.Name, "*UnknownPool*", sizeof(pr.Name));
}
mr.PoolId = jcr->jr.PoolId;
get_job_storage(&store, job, run);
set_storageid_in_mr(store.store, &mr);
/* no need to set ScratchPoolId, since we use fnv_no_create_vol */
if (!find_next_volume_for_append(jcr, &mr, 1, fnv_no_create_vol, fnv_prune)) {
ua->error_msg(_("Could not find next Volume for Job %s (Pool=%s, Level=%s).\n"),
job->name(), pr.Name, level_to_str(run->level));
} else {
ua->send_msg(
_("The next Volume to be used by Job \"%s\" (Pool=%s, Level=%s) will be %s\n"),
job->name(), pr.Name, level_to_str(run->level), mr.VolumeName);
found = true;
}
}
get_out:
if (jcr->db) {
db_sql_close_pooled_connection(jcr, jcr->db);
jcr->db = NULL;
}
free_jcr(jcr);
if (!found) {
ua->error_msg(_("Could not find next Volume for Job %s.\n"),
job->hdr.name);
return false;
}
return true;
}
/*
* Handle Director User Agent commands
*/
static void *handle_UA_client_request(void *arg)
{
int status;
UAContext *ua;
JCR *jcr;
BSOCK *user = (BSOCK *)arg;
pthread_detach(pthread_self());
jcr = new_control_jcr("-Console-", JT_CONSOLE);
ua = new_ua_context(jcr);
ua->UA_sock = user;
set_jcr_in_tsd(INVALID_JCR);
user->recv(); /* Get first message */
if (!authenticate_user_agent(ua)) {
goto getout;
}
while (!ua->quit) {
if (ua->api) {
user->signal(BNET_MAIN_PROMPT);
}
status = user->recv();
if (status >= 0) {
pm_strcpy(ua->cmd, ua->UA_sock->msg);
parse_ua_args(ua);
if (ua->argc > 0 && ua->argk[0][0] == '.') {
do_a_dot_command(ua);
} else {
do_a_command(ua);
}
dequeue_messages(ua->jcr);
if (!ua->quit) {
if (console_msg_pending && acl_access_ok(ua, Command_ACL, "messages")) {
if (ua->auto_display_messages) {
pm_strcpy(ua->cmd, "messages");
qmessages_cmd(ua, ua->cmd);
ua->user_notified_msg_pending = false;
} else if (!ua->gui && !ua->user_notified_msg_pending && console_msg_pending) {
if (ua->api) {
user->signal(BNET_MSGS_PENDING);
} else {
bsendmsg(ua, _("You have messages.\n"));
}
ua->user_notified_msg_pending = true;
}
}
if (!ua->api) {
user->signal(BNET_EOD); /* send end of command */
}
}
} else if (is_bnet_stop(user)) {
ua->quit = true;
} else { /* signal */
user->signal(BNET_POLL);
}
}
getout:
close_db(ua);
free_ua_context(ua);
free_jcr(jcr);
user->close();
delete user;
return NULL;
}
/*
* Director requests us to start a job
* Basic tasks done here:
* - We pickup the JobId to be run from the Director.
* - We pickup the device, media, and pool from the Director
* - Wait for a connection from the File Daemon (FD)
* - Accept commands from the FD (i.e. run the job)
* - Return when the connection is terminated or
* there is an error.
*/
bool job_cmd(JCR *jcr)
{
int32_t JobId;
char auth_key[MAX_NAME_LENGTH];
char seed[MAX_NAME_LENGTH];
char spool_size[MAX_NAME_LENGTH];
BSOCK *dir = jcr->dir_bsock;
POOL_MEM job_name, client_name, job, fileset_name, fileset_md5, backup_format;
int32_t JobType, level, spool_attributes, no_attributes, spool_data;
int32_t PreferMountedVols, rerunning, protocol, dumplevel;
int status;
uint64_t quota = 0;
JCR *ojcr;
/*
* Get JobId and permissions from Director
*/
Dmsg1(100, "<dird: %s", dir->msg);
bstrncpy(spool_size, "0", sizeof(spool_size));
status = sscanf(dir->msg, jobcmd, &JobId, job.c_str(), job_name.c_str(),
client_name.c_str(), &JobType, &level, fileset_name.c_str(),
&no_attributes, &spool_attributes, fileset_md5.c_str(),
&spool_data, &PreferMountedVols, spool_size, &rerunning,
&jcr->VolSessionId, &jcr->VolSessionTime, "a, &protocol,
backup_format.c_str(), &dumplevel);
if (status != 20) {
pm_strcpy(jcr->errmsg, dir->msg);
dir->fsend(BAD_job, status, jcr->errmsg);
Dmsg1(100, ">dird: %s", dir->msg);
jcr->setJobStatus(JS_ErrorTerminated);
return false;
}
jcr->rerunning = (rerunning) ? true : false;
jcr->setJobProtocol(protocol);
Dmsg4(100, "rerunning=%d VolSesId=%d VolSesTime=%d Protocol=%d\n",
jcr->rerunning, jcr->VolSessionId, jcr->VolSessionTime, jcr->getJobProtocol());
/*
* Since this job could be rescheduled, we
* check to see if we have it already. If so
* free the old jcr and use the new one.
*/
ojcr = get_jcr_by_full_name(job.c_str());
if (ojcr && !ojcr->authenticated) {
Dmsg2(100, "Found ojcr=0x%x Job %s\n", (unsigned)(intptr_t)ojcr, job.c_str());
free_jcr(ojcr);
}
jcr->JobId = JobId;
Dmsg2(800, "Start JobId=%d %p\n", JobId, jcr);
/*
* If job rescheduled because previous was incomplete,
* the Resched flag is set and VolSessionId and VolSessionTime
* are given to us (same as restarted job).
*/
if (!jcr->rerunning) {
jcr->VolSessionId = newVolSessionId();
jcr->VolSessionTime = VolSessionTime;
}
bstrncpy(jcr->Job, job, sizeof(jcr->Job));
unbash_spaces(job_name);
jcr->job_name = get_pool_memory(PM_NAME);
pm_strcpy(jcr->job_name, job_name);
unbash_spaces(client_name);
jcr->client_name = get_pool_memory(PM_NAME);
pm_strcpy(jcr->client_name, client_name);
unbash_spaces(fileset_name);
jcr->fileset_name = get_pool_memory(PM_NAME);
pm_strcpy(jcr->fileset_name, fileset_name);
jcr->setJobType(JobType);
jcr->setJobLevel(level);
jcr->no_attributes = no_attributes;
jcr->spool_attributes = spool_attributes;
jcr->spool_data = spool_data;
jcr->spool_size = str_to_int64(spool_size);
jcr->fileset_md5 = get_pool_memory(PM_NAME);
pm_strcpy(jcr->fileset_md5, fileset_md5);
jcr->PreferMountedVols = PreferMountedVols;
jcr->RemainingQuota = quota;
unbash_spaces(backup_format);
jcr->backup_format = get_pool_memory(PM_NAME);
pm_strcpy(jcr->backup_format, backup_format);
jcr->DumpLevel = dumplevel;
jcr->authenticated = false;
Dmsg1(50, "Quota set as %llu\n", quota);
/*
* Pass back an authorization key for the File daemon
*/
bsnprintf(seed, sizeof(seed), "%p%d", jcr, JobId);
make_session_key(auth_key, seed, 1);
dir->fsend(OK_job, jcr->VolSessionId, jcr->VolSessionTime, auth_key);
Dmsg2(50, ">dird jid=%u: %s", (uint32_t)jcr->JobId, dir->msg);
jcr->sd_auth_key = bstrdup(auth_key);
memset(auth_key, 0, sizeof(auth_key));
dispatch_new_plugin_options(jcr);
generate_plugin_event(jcr, bsdEventJobStart, (void *)"JobStart");
return true;
}
