struct recipe *recipe_read_from_file(char *filename)
{
struct recipe *recipe=NULL;
unsigned char *buffer;
int fd=open(filename,O_RDONLY);
if (fd==-1) {
snprintf(recipe_error,1024,"Could not open recipe file '%s'\n",filename);
return NULL;
}
struct stat stat;
if (fstat(fd, &stat) == -1) {
snprintf(recipe_error,1024,"Could not stat recipe file '%s'\n",filename);
close(fd); return NULL;
}
buffer=mmap(NULL, stat.st_size, PROT_READ, MAP_SHARED, fd, 0);
if (buffer==MAP_FAILED) {
snprintf(recipe_error,1024,"Could not memory map recipe file '%s'\n",filename);
close(fd); return NULL;
}
recipe=recipe_read(filename,(char *)buffer,stat.st_size);
munmap(buffer,stat.st_size);
close(fd);
if (recipe&&recipe->field_count==0) {
recipe_free(recipe);
snprintf(recipe_error,1024,"Recipe contains no field definitions\n");
return NULL;
}
return recipe;
}
struct recipe *recipe_read(char *formname,char *buffer,int buffer_size)
{
if (buffer_size<1||buffer_size>1048576) {
snprintf(recipe_error,1024,"Recipe file empty or too large (>1MB).\n");
return NULL;
}
struct recipe *recipe=calloc(sizeof(struct recipe),1);
if (!recipe) {
snprintf(recipe_error,1024,"Allocation of recipe structure failed.\n");
return NULL;
}
// Get recipe hash
recipe_form_hash(formname,recipe->formhash,recipe->formname);
LOGI("recipe_read(): Computing formhash based on form name '%s'",formname);
int i;
int l=0;
int line_number=1;
char line[16384];
char name[16384],type[16384];
int min,max,precision;
char enumvalues[16384];
for(i=0;i<=buffer_size;i++) {
if (l>16380) {
snprintf(recipe_error,1024,"line:%d:Line too long.\n",line_number);
recipe_free(recipe); return NULL; }
if ((i==buffer_size)||(buffer[i]=='\n')||(buffer[i]=='\r')) {
if (recipe->field_count>1000) {
snprintf(recipe_error,1024,"line:%d:Too many field definitions (must be <=1000).\n",line_number);
recipe_free(recipe); return NULL;
}
// Process recipe line
line[l]=0;
if ((l>0)&&(line[0]!='#')) {
enumvalues[0]=0;
if (sscanf(line,"%[^:]:%[^:]:%d:%d:%d:%[^\n]",
name,type,&min,&max,&precision,enumvalues)>=5) {
int fieldtype=recipe_parse_fieldtype(type);
if (fieldtype==-1) {
snprintf(recipe_error,1024,"line:%d:Unknown or misspelled field type '%s'.\n",line_number,type);
recipe_free(recipe); return NULL;
} else {
// Store parsed field
recipe->fields[recipe->field_count].name=strdup(name);
recipe->fields[recipe->field_count].type=fieldtype;
recipe->fields[recipe->field_count].minimum=min;
recipe->fields[recipe->field_count].maximum=max;
recipe->fields[recipe->field_count].precision=precision;
if (fieldtype==FIELDTYPE_ENUM||fieldtype==FIELDTYPE_MULTISELECT) {
char enum_value[1024];
int e=0;
int en=0;
int i;
for(i=0;i<=strlen(enumvalues);i++) {
if ((enumvalues[i]==',')||(enumvalues[i]==0)) {
// End of field
enum_value[e]=0;
if (en>=MAX_ENUM_VALUES) {
snprintf(recipe_error,1024,"line:%d:enum has too many values (max=32)\n",line_number);
recipe_free(recipe);
return NULL;
}
recipe->fields[recipe->field_count].enum_values[en]
=strdup(enum_value);
en++;
e=0;
} else {
// next character of field
enum_value[e++]=enumvalues[i];
}
}
if (en<1) {
snprintf(recipe_error,1024,"line:%d:Malformed enum field definition: must contain at least one value option.\n",line_number);
recipe_free(recipe); return NULL;
}
recipe->fields[recipe->field_count].enum_count=en;
}
recipe->field_count++;
}
} else {
snprintf(recipe_error,1024,"line:%d:Malformed field definition.\n",line_number);
recipe_free(recipe); return NULL;
}
}
line_number++; l=0;
} else {
line[l++]=buffer[i];
}
}
return recipe;
}
JNIEXPORT jobjectArray JNICALL Java_org_servalproject_succinctdata_jni_xml2succinctfragments
(JNIEnv * env, jobject jobj,
jstring xmlforminstance,
jstring xmlformspecification,
jstring formname,
jstring formversion,
jstring succinctpath,
jstring smacdat,
jint mtu,
jint debug)
{
LOGI(" xml2succinctfragments ENTRY");
const char *xmldata= (*env)->GetStringUTFChars(env,xmlforminstance,0);
LOGI(" xml2succinctfragments E2");
const char *formname_c= NULL; // (*env)->GetStringUTFChars(env,formname,0);
LOGI(" xml2succinctfragments E3");
const char *formversion_c= NULL; // (*env)->GetStringUTFChars(env,formversion,0);
LOGI(" xml2succinctfragments E4");
const char *path= NULL; // (*env)->GetStringUTFChars(env,succinctpath,0);
LOGI(" xml2succinctfragments E5");
const char *xmlform_c= (*env)->GetStringUTFChars(env,xmlformspecification,0);
LOGI(" xml2succinctfragments E6");
const char *smacdat_c= (*env)->GetStringUTFChars(env,smacdat,0);
LOGI(" xml2succinctfragments E7");
char stripped[65536];
unsigned char succinct[1024];
int succinct_len=0;
char filename[1024];
int magpi_mode=0;
LOGI(" xml2succinctfragments checkpoing 1");
// Automatically detect Magpi forms versus ODK ones.
// Magpi forms are HTML documents, where as ODK uses XML ones.
if (xmlform_c&&(!strncasecmp("<html",xmlform_c,5))) magpi_mode=1;
// Read public key hex
LOGI(" xml2succinctfragments checkpoing 2: magpi_mode = %d",magpi_mode);
// Default to public key of Serval succinct data server
char publickeyhex[1024]="74f3a36029b0e60084d42bd9cafa3f2b26fe802b0a6f024ff00451481c9bba4a";
if (path&&formname_c&&formversion_c)
{
LOGI("recipe: Expecting public key file");
snprintf(filename,1024,"%s/%s.%s.publickey",path,formname_c,formversion_c);
LOGI("recipe: Opening recipient public key file %s",filename);
FILE *f=fopen(filename,"r");
if (f) {
int r=fread(publickeyhex,1,1023,f);
if (r<64) {
char message[1024];
snprintf(message,1024,"Failed to read from public key file");
return error_message(env,message);
LOGI("recipe: failed to load publickeyhex from file (using default value)");
}
publickeyhex[r]=0;
}
// Trim CR/LF from end
if (publickeyhex[0]) {
while(publickeyhex[strlen(publickeyhex)-1]<' ')
publickeyhex[strlen(publickeyhex)-1]=0;
}
fclose(f);
}
LOGI("recipe: have publixkeyhex = '%s'",publickeyhex);
struct recipe *recipe=NULL;
if (xmlformspecification==NULL||xmlform_c==NULL||!strlen(xmlform_c)) {
// Read recipe file
snprintf(filename,1024,"%s/%s.%s.recipe",path,formname_c,formversion_c);
LOGI("Opening recipe file %s",filename);
recipe=recipe_read_from_file(filename);
if (!recipe) {
char message[1024];
snprintf(message,1024,"Could not read recipe file %s",filename);
return error_message(env,message);
}
} else {
// Create recipe from form specification
char recipetext[65536];
int recipetextLen=65536;
char templatetext[65536];
int templatetextLen=65536;
char the_form_name[1024];
char the_form_version[1024];
LOGI("Using form specification to create recipe on the fly (magpi_mode=%d, form spec = %d bytes)",
magpi_mode,strlen(xmlform_c));
LOGI("Form specification is: %s",xmlform_c);
int r;
if (magpi_mode) {
r=xhtmlToRecipe(xmlform_c,strlen(xmlform_c),
the_form_name,the_form_version,
recipetext,&recipetextLen,
templatetext,&templatetextLen);
// Magpi forms are identified solely by the numeric formid
strcpy(the_form_name,the_form_version);
strcpy(the_form_version,"this should not be used");
}
else
r=xmlToRecipe(xmlform_c,strlen(xmlform_c),
the_form_name,the_form_version,
recipetext,&recipetextLen,
templatetext,&templatetextLen);
if (r) {
return error_message(env,"Could not create recipe from form specification");
}
LOGI("Have %d bytes of recipe text (name='%s', version='%s')",
recipetextLen,the_form_name,the_form_version);
if (recipetextLen<10) return error_message(env,"Could not convert form specification to recipe");
formname_c=form_name; formversion_c=form_version;
recipe = recipe_read(the_form_name,recipetext,recipetextLen);
if (!recipe) {
char message[1024];
snprintf(message,1024,"Could not set recipe");
LOGI("Failed to read recipe from buffer");
LOGI("Recipe is:\n%s",recipetext);
return error_message(env,message);
}
LOGI("Read recipe from buffer (%d bytes). Hash = %02x%02x%02x%02x%02x%02x",
recipetextLen,
recipe->formhash[0],recipe->formhash[1],recipe->formhash[2],
recipe->formhash[3],recipe->formhash[4],recipe->formhash[5]
);
LOGI("Recipe is:\n%s\n",recipetext);
}
LOGI("About to run xml2stripped");
// Transform XML to stripped data first.
int stripped_len;
stripped_len = xml2stripped(formname_c,xmldata,strlen(xmldata),stripped,sizeof(stripped));
LOGI("Stripped data is %d bytes long",stripped_len);
if (stripped_len>0) {
// Produce succinct data
LOGI("About to read stats file %s",smacdat_c);
// Get stats handle
stats_handle *h=stats_new_handle(smacdat_c);
stats_load_tree(h);
LOGI("Loaded entire stats tree");
if (!h) {
recipe_free(recipe);
char message[1024];
snprintf(message,1024,"Could not read SMAC stats file %s",smacdat_c);
return error_message(env,message);
}
LOGI("Read stats, now about to call recipe_compresS()");
// Compress stripped data to form succinct data
succinct_len=recipe_compress(h,recipe,stripped,stripped_len,succinct,sizeof(succinct));
LOGI("Binary succinct data is %d bytes long",succinct_len);
// Clean up after ourselves
stats_handle_free(h);
recipe_free(recipe);
char filename[1024];
snprintf(filename,1024,"%s/%s.%s.recipe",path,formname_c,formversion_c);
LOGI("Cleaned up after ourselves");
if (succinct_len<1) {
LOGI("Failed to compress XML - reporting error and exiting");
char message[1024];
snprintf(message,1024,"recipe_compess failed with recipe file %s. stripped_len=%d",filename,stripped_len);
LOGI("Exiting due to failure to produce valid Succinct Data output.");
return error_message(env,message);
}
} else {
LOGI("Failed to strop XML using recipe file -- exiting.");
recipe_free(recipe);
char message[1024];
snprintf(message,1024,"Failed to strip XML using recipe file %s.",filename);
return error_message(env,message);
}
LOGI("Fragmenting succinct data record");
char *fragments[MAX_FRAGMENTS];
int fragment_count=0;
encryptAndFragmentBuffer(succinct,succinct_len,fragments,&fragment_count,mtu,
publickeyhex,debug);
LOGI("Succinct data formed into %d fragments",fragment_count);
jobjectArray result=
(jobjectArray)(*env)->NewObjectArray(env,fragment_count,
(*env)->FindClass(env,"java/lang/String"),
(*env)->NewStringUTF(env,""));
for(int i=0;i<fragment_count;i++) {
(*env)->SetObjectArrayElement(env,result,i,(*env)->NewStringUTF(env,fragments[i]));
free(fragments[i]);
}
return result;
}
int backup(Jcr* jcr) {
fingerchunk_queue = sync_queue_new(-1);
ContainerUsageMonitor* usage_monitor = container_usage_monitor_new();
cfl_monitor = cfl_monitor_new(read_cache_size);
if (simulation_level == SIMULATION_ALL) {
start_read_trace_phase(jcr);
} else {
start_read_phase(jcr);
start_chunk_phase(jcr);
start_hash_phase(jcr);
}
start_segment_phase(jcr);
start_filter_phase(jcr);
start_append_phase(jcr);
ContainerId seed_id = -1;
int32_t seed_len = 0;
FingerChunk* fchunk = NULL;
int signal = recv_fingerchunk(&fchunk);
while (signal != STREAM_END) {
container_usage_monitor_update(usage_monitor, fchunk->container_id,
&fchunk->fingerprint, fchunk->length);
jvol_append_fingerchunk(jcr->job_volume, fchunk);
if (seed_id != -1 && seed_id != fchunk->container_id) {
jvol_append_seed(jcr->job_volume, seed_id, seed_len);
seed_len = 0;
}
/* merge sequential accesses */
seed_id = fchunk->container_id;
seed_len += fchunk->length;
free(fchunk);
signal = recv_fingerchunk(&fchunk);
}
if (seed_len > 0)
jvol_append_seed(jcr->job_volume, seed_id, seed_len);
sync_queue_free(fingerchunk_queue, NULL);
jcr->sparse_container_num = g_hash_table_size(usage_monitor->sparse_map);
jcr->total_container_num = g_hash_table_size(usage_monitor->dense_map)
+ jcr->sparse_container_num;
while ((jcr->inherited_sparse_num = container_usage_monitor_print(
usage_monitor, jcr->job_id, jcr->historical_sparse_containers)) < 0) {
dprint("retry!");
}
/* store recipes of processed file */
int i = 0;
for (; i < jcr->file_num; ++i) {
Recipe *recipe = (Recipe*) sync_queue_pop(jcr->completed_files_queue);
recipe->fileindex = i;
if (jvol_append_meta(jcr->job_volume, recipe) != SUCCESS) {
printf("%s, %d: some errors happened in appending recipe!\n",
__FILE__, __LINE__);
return FAILURE;
}
jcr->chunk_num += recipe->chunknum;
recipe_free(recipe);
}
stop_append_phase();
stop_filter_phase();
stop_segment_phase();
if (simulation_level == SIMULATION_ALL) {
stop_read_trace_phase(jcr);
} else {
stop_hash_phase();
stop_chunk_phase();
stop_read_phase();
}
container_usage_monitor_free(usage_monitor);
print_cfl(cfl_monitor);
cfl_monitor_free(cfl_monitor);
return 0;
}
void backup_formal(int fd,char *msg){
JCR *jcr=NULL;
char fileset[256]={0};
char *buf=(char *)calloc(1,SOCKET_BUF_SIZE+21);
int len;
int index=1;
char vol_name[FILE_NAME_LEN];
int vol_fd;
Recipe *rp=NULL;
FingerChunk *fc=NULL;
char *p=NULL;
int64_t rwlen=0;
jobcount_init();
jcr=jcr_new();
jcr->dataSocket=fd;
memset(vol_name,0,FILE_NAME_LEN);
strcpy(vol_name,BackupVolPath);
strcat(vol_name,"data_vol");
vol_fd=open(vol_name,O_RDWR| O_CREAT,00644);
if(vol_fd<0){
err_msg1("can't open file");
goto FAIL;
}
printf("%s %d vol_name:%s\n",__FILE__,__LINE__,vol_name);
rwlen=lseek(vol_fd,0,SEEK_END);
TIMER_DECLARE(gstart,gend);
TIMER_DECLARE(wstart,wend);
TIMER_START(gstart);
if(sscanf(msg,backup_cmd,fileset)!=1){ // backup cmd
goto FAIL;
}
jcr->jobv=jobv_new(fileset);
jcr->nJobId=jcr->jobv->nJobId;
printf("===========backup start==============\n");
printf("%s,%d pathname:%s \n", __FILE__,__LINE__,fileset);
while(bnet_recv(jcr->dataSocket,buf,&len)!=ERROR){ //文件名
if(len==STREAM_END){
printf("%s %d backup is over\n",__FILE__,__LINE__);
break;
}
//printf("\033[40;32m recv file: %s (%d) \033[0m\n",buf,len);
rp=recipe_new();
memcpy(rp->filename,buf,len);
rp->fileindex=index++;
while(bnet_recv(jcr->dataSocket,buf,&len)>0){ /*format: fingerprintf data data dta..*/
//printf("\033[40;32m recv: file data (%d) \033[0m\n",len);
fc=fingerchunk_new(buf,0);
fc->offset=rwlen;
fc->length=len-sizeof(Fingerprint);
check_data(fc->fingerprint,buf+sizeof(Fingerprint),fc->length);
TIMER_START(wstart);
if(writen(vol_fd,buf+sizeof(Fingerprint),fc->length)!=fc->length)
err_msg1("wrintn wrong");
TIMER_END(wend);
TIMER_DIFF(jcr->writeDataTime,wstart,wend);
rwlen+=fc->length;
jcr->nChunkCount++;
jcr->nSize+=fc->length;
recipe_append_fingerchunk(rp,fc);
}
jcr->nFileCount++;
if(G_VERBOSE)
printf("receive file %s OK, total: %d\n",rp->filename,jcr->nFileCount);
jobv_insert_recipe(jcr->jobv, rp);
rp=NULL;
}
FAIL:
bnet_send(fd,"OK",2); // 发送备份成功信息
TIMER_END(gend);
TIMER_DIFF(jcr->recvTime,gstart,gend);
printf("============back over===============\n");
printf("total time:%.4f %.4f MB/s\n",jcr->recvTime,jcr->nSize*1.0/jcr->recvTime/1036288.0);
printf("write time:%.4f %.4f MB/s\n",jcr->writeDataTime,jcr->nSize*1.0/jcr->writeDataTime/1036288.0);
printf("chunk count:%d\n",jcr->nChunkCount);
printf("file count:%d\n",jcr->nFileCount);
if(rp){
recipe_free(rp);
}
jobv_destroy(jcr->jobv);
jcr_free(jcr);
jobcount_close();
close(vol_fd);
}
void restore_formal(int fd,char *msg){
JCR *jcr=NULL;
Recipe *rp=NULL;
FingerChunk *fc=NULL;
char *buf=(char *)calloc(1,SOCKET_BUF_SIZE+21);
char vol_name[FILE_NAME_LEN];
int vol_fd;
//char stream[30]={0};
memset(vol_name,0,FILE_NAME_LEN);
strcpy(vol_name,BackupVolPath);
strcat(vol_name,"data_vol");
vol_fd=open(vol_name,O_RDWR| O_CREAT,00644);
jcr=jcr_new();
jcr->dataSocket=fd;
if(vol_fd<0){
err_msg1("can't open file");
goto FAIL;
}
if(sscanf(msg,restore_cmd,&jcr->nJobId)!=1){ // backup cmd
goto FAIL;
}
jcr->jobv=jobv_open(jcr->nJobId);
if(jcr->jobv==NULL){
goto FAIL;
}
printf("%s,%d restore jobid:%d \n", __FILE__,__LINE__,jcr->nJobId);
// send pathname
bnet_send(fd,jcr->jobv->szBackupPath,strlen(jcr->jobv->szBackupPath));
while((rp=jobv_search_next_recipe(jcr->jobv))){
//send file name
if(G_VERBOSE)
printf("send file %s \n",rp->filename);
bnet_send(fd,rp->filename,strlen(rp->filename));
//send data
for(fc=rp->first;fc;fc=fc->next){
lseek(vol_fd,fc->offset,SEEK_SET);
if(readn(vol_fd,buf,fc->length)!=fc->length)
err_msg1("readn wrong");
check_data(fc->fingerprint,buf,fc->length);
bnet_send(fd,buf,fc->length);
}
bnet_signal(fd,DATA_END);
recipe_free(rp);
}
FAIL:
bnet_signal(fd,STREAM_END); /* over */
jobv_destroy(jcr->jobv);
jcr_free(jcr);
}
