static void *do_batch(void *jcr)
{
JCR *bjcr = (JCR *)jcr;
char data[1024];
int lineno = 0;
struct ATTR_DBR ar;
memset(&ar, 0, sizeof(ar));
btime_t begin = get_current_btime();
char *datafile = bjcr->where;
FILE *fd = fopen(datafile, "r");
if (!fd) {
Emsg1(M_ERROR_TERM, 0, _("Error opening datafile %s\n"), datafile);
}
while (fgets(data, sizeof(data)-1, fd)) {
strip_trailing_newline(data);
lineno++;
if (verbose && ((lineno % 5000) == 1)) {
printf("\r%i", lineno);
}
fill_attr(&ar, data);
if (!db_create_attributes_record(bjcr, bjcr->db, &ar)) {
Emsg0(M_ERROR_TERM, 0, _("Error while inserting file\n"));
}
}
fclose(fd);
db_write_batch_file_records(bjcr);
btime_t end = get_current_btime();
P(mutex);
char ed1[200], ed2[200];
printf("\rbegin = %s, end = %s\n", edit_int64(begin, ed1),edit_int64(end, ed2));
printf("Insert time = %sms\n", edit_int64((end - begin) / 10000, ed1));
printf("Create %u files at %.2f/s\n", lineno,
(lineno / ((float)((end - begin) / 1000000))));
nb--;
V(mutex);
pthread_exit(NULL);
return NULL;
}
/*
* Note, we receive the whole attribute record, but we select out only the stat
* packet, VolSessionId, VolSessionTime, FileIndex, file type, and file name to
* store in the catalog.
*/
static void update_attribute(JCR *jcr, char *msg, int32_t msglen)
{
unser_declare;
uint32_t VolSessionId, VolSessionTime;
int32_t Stream;
uint32_t FileIndex;
char *p;
int len;
char *fname, *attr;
ATTR_DBR *ar = NULL;
uint32_t reclen;
/*
* Start transaction allocates jcr->attr and jcr->ar if needed
*/
db_start_transaction(jcr, jcr->db); /* start transaction if not already open */
ar = jcr->ar;
/*
* Start by scanning directly in the message buffer to get Stream
* there may be a cached attr so we cannot yet write into
* jcr->attr or jcr->ar
*/
p = msg;
skip_nonspaces(&p); /* UpdCat */
skip_spaces(&p);
skip_nonspaces(&p); /* Job=nnn */
skip_spaces(&p);
skip_nonspaces(&p); /* "FileAttributes" */
p += 1;
/*
* The following "SD header" fields are serialized
*/
unser_begin(p, 0);
unser_uint32(VolSessionId); /* VolSessionId */
unser_uint32(VolSessionTime); /* VolSessionTime */
unser_int32(FileIndex); /* FileIndex */
unser_int32(Stream); /* Stream */
unser_uint32(reclen); /* Record length */
p += unser_length(p); /* Raw record follows */
/**
* At this point p points to the raw record, which varies according
* to what kind of a record (Stream) was sent. Note, the integer
* fields at the beginning of these "raw" records are in ASCII with
* spaces between them so one can use scanf or manual scanning to
* extract the fields.
*
* File Attributes
* File_index
* File type
* Filename (full path)
* Encoded attributes
* Link name (if type==FT_LNK or FT_LNKSAVED)
* Encoded extended-attributes (for Win32)
* Delta sequence number (32 bit int)
*
* Restore Object
* File_index
* File_type
* Object_index
* Object_len (possibly compressed)
* Object_full_len (not compressed)
* Object_compression
* Plugin_name
* Object_name
* Binary Object data
*/
Dmsg1(400, "UpdCat msg=%s\n", msg);
Dmsg5(400, "UpdCat VolSessId=%d VolSessT=%d FI=%d Strm=%d reclen=%d\n",
VolSessionId, VolSessionTime, FileIndex, Stream, reclen);
jcr->SDJobBytes += reclen; /* update number of bytes transferred for quotas */
/*
* Depending on the stream we are handling dispatch.
*/
switch (Stream) {
case STREAM_UNIX_ATTRIBUTES:
case STREAM_UNIX_ATTRIBUTES_EX:
if (jcr->cached_attribute) {
Dmsg2(400, "Cached attr. Stream=%d fname=%s\n", ar->Stream, ar->fname);
if (!db_create_attributes_record(jcr, jcr->db, ar)) {
Jmsg1(jcr, M_FATAL, 0, _("Attribute create error: ERR=%s"), db_strerror(jcr->db));
}
jcr->cached_attribute = false;
}
/*
* Any cached attr is flushed so we can reuse jcr->attr and jcr->ar
*/
jcr->attr = check_pool_memory_size(jcr->attr, msglen);
memcpy(jcr->attr, msg, msglen);
p = jcr->attr - msg + p; /* point p into jcr->attr */
skip_nonspaces(&p); /* skip FileIndex */
skip_spaces(&p);
ar->FileType = str_to_int32(p);
skip_nonspaces(&p); /* skip FileType */
skip_spaces(&p);
fname = p;
len = strlen(fname); /* length before attributes */
attr = &fname[len+1];
ar->DeltaSeq = 0;
if (ar->FileType == FT_REG) {
p = attr + strlen(attr) + 1; /* point to link */
p = p + strlen(p) + 1; /* point to extended attributes */
p = p + strlen(p) + 1; /* point to delta sequence */
/*
* Older FDs don't have a delta sequence, so check if it is there
*/
if (p - jcr->attr < msglen) {
ar->DeltaSeq = str_to_int32(p); /* delta_seq */
}
}
Dmsg2(400, "dird<stored: stream=%d %s\n", Stream, fname);
Dmsg1(400, "dird<stored: attr=%s\n", attr);
ar->attr = attr;
ar->fname = fname;
if (ar->FileType == FT_DELETED) {
ar->FileIndex = 0; /* special value */
} else {
ar->FileIndex = FileIndex;
}
ar->Stream = Stream;
ar->link = NULL;
if (jcr->mig_jcr) {
ar->JobId = jcr->mig_jcr->JobId;
} else {
ar->JobId = jcr->JobId;
}
ar->Digest = NULL;
ar->DigestType = CRYPTO_DIGEST_NONE;
jcr->cached_attribute = true;
Dmsg2(400, "dird<filed: stream=%d %s\n", Stream, fname);
Dmsg1(400, "dird<filed: attr=%s\n", attr);
break;
case STREAM_RESTORE_OBJECT: {
ROBJECT_DBR ro;
memset(&ro, 0, sizeof(ro));
ro.Stream = Stream;
ro.FileIndex = FileIndex;
if (jcr->mig_jcr) {
ro.JobId = jcr->mig_jcr->JobId;
} else {
ro.JobId = jcr->JobId;
}
Dmsg1(100, "Robj=%s\n", p);
skip_nonspaces(&p); /* skip FileIndex */
skip_spaces(&p);
ro.FileType = str_to_int32(p); /* FileType */
skip_nonspaces(&p);
skip_spaces(&p);
ro.object_index = str_to_int32(p); /* Object Index */
skip_nonspaces(&p);
skip_spaces(&p);
ro.object_len = str_to_int32(p); /* object length possibly compressed */
skip_nonspaces(&p);
skip_spaces(&p);
ro.object_full_len = str_to_int32(p); /* uncompressed object length */
skip_nonspaces(&p);
skip_spaces(&p);
ro.object_compression = str_to_int32(p); /* compression */
skip_nonspaces(&p);
skip_spaces(&p);
ro.plugin_name = p; /* point to plugin name */
len = strlen(ro.plugin_name);
ro.object_name = &ro.plugin_name[len+1]; /* point to object name */
len = strlen(ro.object_name);
ro.object = &ro.object_name[len+1]; /* point to object */
ro.object[ro.object_len] = 0; /* add zero for those who attempt printing */
Dmsg7(100, "oname=%s stream=%d FT=%d FI=%d JobId=%d, obj_len=%d\nobj=\"%s\"\n",
ro.object_name, ro.Stream, ro.FileType, ro.FileIndex, ro.JobId,
ro.object_len, ro.object);
/*
* Store it.
*/
if (!db_create_restore_object_record(jcr, jcr->db, &ro)) {
Jmsg1(jcr, M_FATAL, 0, _("Restore object create error. %s"), db_strerror(jcr->db));
}
break;
}
default:
if (crypto_digest_stream_type(Stream) != CRYPTO_DIGEST_NONE) {
fname = p;
if (ar->FileIndex != FileIndex) {
Jmsg3(jcr, M_WARNING, 0, _("%s not same File=%d as attributes=%d\n"),
stream_to_ascii(Stream), FileIndex, ar->FileIndex);
} else {
/*
* Update digest in catalog
*/
char digestbuf[BASE64_SIZE(CRYPTO_DIGEST_MAX_SIZE)];
int len = 0;
int type = CRYPTO_DIGEST_NONE;
switch(Stream) {
case STREAM_MD5_DIGEST:
len = CRYPTO_DIGEST_MD5_SIZE;
type = CRYPTO_DIGEST_MD5;
break;
case STREAM_SHA1_DIGEST:
len = CRYPTO_DIGEST_SHA1_SIZE;
type = CRYPTO_DIGEST_SHA1;
break;
case STREAM_SHA256_DIGEST:
len = CRYPTO_DIGEST_SHA256_SIZE;
type = CRYPTO_DIGEST_SHA256;
break;
case STREAM_SHA512_DIGEST:
len = CRYPTO_DIGEST_SHA512_SIZE;
type = CRYPTO_DIGEST_SHA512;
break;
default:
/*
* Never reached ...
*/
Jmsg(jcr, M_ERROR, 0, _("Catalog error updating file digest. Unsupported digest stream type: %d"), Stream);
}
bin_to_base64(digestbuf, sizeof(digestbuf), fname, len, true);
Dmsg3(400, "DigestLen=%d Digest=%s type=%d\n", strlen(digestbuf),
digestbuf, Stream);
if (jcr->cached_attribute) {
ar->Digest = digestbuf;
ar->DigestType = type;
Dmsg2(400, "Cached attr with digest. Stream=%d fname=%s\n", ar->Stream, ar->fname);
/*
* Update BaseFile table
*/
if (!db_create_attributes_record(jcr, jcr->db, ar)) {
Jmsg1(jcr, M_FATAL, 0, _("attribute create error. %s"), db_strerror(jcr->db));
}
jcr->cached_attribute = false;
} else {
if (!db_add_digest_to_file_record(jcr, jcr->db, ar->FileId, digestbuf, type)) {
Jmsg(jcr, M_ERROR, 0, _("Catalog error updating file digest. %s"), db_strerror(jcr->db));
}
}
}
}
break;
}
}
