int build_index(UCHAR *text, ULONG length, char *build_options, void **index){
char delimiters[] = " =;";
char filename[256];
int j,num_parameters;
char ** parameters;
int rankb_w=16,rankb_w2=128;
int free_text=false; /* don't free text by default */
if (build_options != NULL) {
// printf("build_options=%s\n",build_options);
parse_parameters(build_options,&num_parameters, ¶meters, delimiters);
for (j=0; j<num_parameters;j++) {
if ((strcmp(parameters[j], "samplerate") == 0 ) && (j < num_parameters-1) ) {
rankb_w=atoi(parameters[j+1]);
j++;
} else if ((strcmp(parameters[j], "samplepsi") == 0 ) && (j < num_parameters-1) ) {
rankb_w2=atoi(parameters[j+1]);
j++;
} else if ((strcmp(parameters[j], "filename") == 0 ) && (j < num_parameters-1) ) {
strcpy(filename,parameters[j+1]);
j++;
} else if (strcmp(parameters[j], "free_text") == 0 )
free_text=true;
}
free_parameters(num_parameters, ¶meters);
}
int n=length;
char fname1[128],fname2[128];
/* make the SA */
int i, *x, *p;
int k, l;
p= (int *)malloc((n+1)*sizeof *p);
x= (int *)malloc((n+1)*sizeof *x);
if (! p || ! x) {
return 1;
}
for ( i=0; i<n; i++) {
x[i]=text[i];
}
l=0;
k=UCHAR_MAX+1;
suffixsort(x, p, n, k, l);
free(x);
p[0] = n;
/* End Make SA */
/* ³Ì¶ñÍ 0..n-1 Å n ÔÚÉÍ 0 ªüéB
p[0] ÍK¸ n ÉÈéBp[1..n]É0..n-1ªüÁÄ¢éB*/
for (i=0; i<=n; ++i) p[i]++; /* p[1..n]É1..nªüÁÄ¢éBp[0]=n+1*/
sprintf(fname1,"%s.psi",filename);
sprintf(fname2,"%s.idx",filename);
csa_new(n,p,text,fname1,fname2,rankb_w,rankb_w2);
free(p);
if (free_text) free(text);
load_index(filename, index);
return 0;
}
/*////////////////////
//Building the Index//
////////////////////*/
int build_index(uchar *text, ulong length, char *build_options, void **index){
/*if (text[length-1]!='\0') return 2;*/
ulong i, *p;
long overshoot;
TSA_Un *_index= (TSA_Un *) malloc(sizeof(TSA_Un));
uchar *x;
char delimiters[] = " =;";
int j,num_parameters;
char ** parameters;
int copy_text=false; /* don't copy text by default */
int free_text=false; /* don't free text by default */
if (!_index) return 1;
if (build_options != NULL) {
parse_parameters(build_options,&num_parameters, ¶meters, delimiters);
for (j=0; j<num_parameters;j++) {
if (strcmp(parameters[j], "copy_text") == 0 )
copy_text=true;
else if (strcmp(parameters[j], "free_text") == 0 )
free_text=true;
}
free_parameters(num_parameters, ¶meters);
}
/* Consistence of parameters */
if ((!copy_text) && (free_text))
return 5;
/* */
if ( !copy_text ) {
_index->text = text;
_index->own=false;
} else {
_index->text = (uchar *) malloc(sizeof(uchar)*length);
if (!_index->text) return 1;
for (i=0;i<length;i++) _index->text[i]=text[i];
_index->own=true;
}
if ( free_text )
free(text);
_index->n=length;
/* Make suffix array */
overshoot = init_ds_ssort(500, 2000);
p= (ulong *) malloc (sizeof(ulong)*(length));
if (!p) return 1;
x= (uchar *) malloc (sizeof(uchar)*(length+overshoot));
if (!x) return 1;
for (i=0;i<length;i++) x[i]=_index->text[i];
ds_ssort( x, p, _index->n);
free(x);
_index->pos = p;
(*index) = _index;
return 0;
}
void ini_close(s_ini_handle* handle)
{
s_ini_section* cur_section;
if (handle == 0)
return;
free_parameters(handle->parameters);
cur_section = handle->sections;
while(cur_section)
{
s_ini_section* next_section = cur_section->next;
free_parameters(cur_section->parameters);
free(cur_section->name);
free(cur_section);
cur_section = next_section;
}
free(handle);
}
void stream(T4C* t4c, sds url, Parameters* paramsArgument, size_t (*callback)(void*, size_t, size_t, void*)) {
Parameters* oauthParams = new_parameters();
genOAuthParams(t4c, oauthParams);
Parameters* params = new_parameters();
buildParams(params, oauthParams, paramsArgument);
sds oauthSignature = signature(t4c->consumerSecret, t4c->accessTokenSecret, GET, url, params);
sds encodedSignature = url_encode(oauthSignature);
add_parameter(oauthParams, sdsnew("oauth_signature"), encodedSignature);
add_parameter(params, sdsnew("oauth_signature"), encodedSignature);
sds authorizeChild = join_parameters(oauthParams, ",");
sds authorize = sdscatprintf(sdsempty(), "Authorization: OAuth %s", authorizeChild);
sds path = join_parameters(params, "&");
if (DEBUG) {
printf("----------------------------\n");
printf("STREAMING API");
printf("URL: %s\n", url);
printf("path: %s\n", path);
printf("authorize: %s\n", authorize);
printf("----------------------------\n");
}
CURL* curl;
curl = curl_easy_init();
sds reqURL = sdscatprintf(sdsempty(), "%s?%s", url, path);
curl_easy_setopt(curl, CURLOPT_URL, reqURL);
struct curl_slist *headers = NULL;
headers = curl_slist_append(headers, authorize);
curl_easy_setopt(curl, CURLOPT_HEADER, headers);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, callback);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 0);
curl_easy_perform(curl);
curl_easy_cleanup(curl);
sdsfree(oauthSignature);
sdsfree(encodedSignature);
sdsfree(reqURL);
sdsfree(url);
sdsfree(path);
sdsfree(authorize);
sdsfree(authorizeChild);
free_parameters(oauthParams);
}
int setDefaultSearchOptions_il (void *ail, char *search_options){
t_il *il = (t_il *) ail;
il->defs.defaultFsearch = NULL;
il->defs.defaultIntersect2 = NULL;
il->defs.defaultIntersectN = NULL;
/** processing the parameters */
char delimiters[] = " =;";
int j,num_parameters;
char ** parameters;
if (search_options != NULL) {
parse_parameters(search_options,&num_parameters, ¶meters, delimiters);
for (j=0; j<num_parameters;j++) {
//setting the default function to Intersect-2-lists
if ((strcmp(parameters[j], "int2") == 0 ) && (j < num_parameters-1) ) {
if (strcmp(parameters[j+1], "svs") == 0 )
il->defs.defaultIntersect2 = svs2;
else if (strcmp(parameters[j+1], "merge") == 0 )
il->defs.defaultIntersect2 = merge2;
j++;
}
//setting the default function to Intersect-N-lists
else if ((strcmp(parameters[j], "intn") == 0 ) && (j < num_parameters-1) ) {
if (strcmp(parameters[j+1], "svs") == 0 )
il->defs.defaultIntersectN = svsn;
else if (strcmp(parameters[j+1], "merge") == 0 )
il->defs.defaultIntersectN = mergeN;
j++;
}
//setting the default fSearch function used in svs-based algoritms.
else if ((strcmp(parameters[j], "fsearch") == 0 ) && (j < num_parameters-1) ) {
if (strcmp(parameters[j+1], "unc") == 0 )
il->defs.defaultFsearch = fsearchUnc;
//else if (strcmp(parameters[j+1], "dec") == 0 )
// il->defs.defaultFsearch = fsearchDec;
//else if (strcmp(parameters[j+1], "seq") == 0 )
// il->defs.defaultFsearch = fsearchSeq;
else if (strcmp(parameters[j+1], "seqM") == 0 )
il->defs.defaultFsearch = fsearchSeqMem;
j++;
}
}
free_parameters(num_parameters, ¶meters);
}
fprintf(stderr,"\n ** Call to setDefaultSearchOptions_il: \"%s\"\n", search_options);
return 0;
}
// Initializes the parameters of the index.
uint parametersCSA(ticsa *myicsa, char *build_options){
char delimiters[] = " =;";
int j,num_parameters;
char ** parameters;
int ssA,ssAinv,ssPsi,nsHuff,psiSearchFactor;
ssA = DEFAULT_A_SAMPLING_PERIOD;
ssAinv = DEFAULT_A_INV_SAMPLING_PERIOD;
ssPsi = DEFAULT_PSI_SAMPLING_PERIOD;
nsHuff = DEFAULT_nsHUFF;
psiSearchFactor = DEFAULT_PSI_BINARY_SEARCH_FACTOR;
if (build_options != NULL) {
parse_parameters(build_options,&num_parameters, ¶meters, delimiters);
for (j=0; j<num_parameters;j++) {
if ((strcmp(parameters[j], "sA") == 0 ) && (j < num_parameters-1) ) {
ssA=atoi(parameters[j+1]);
}
if ((strcmp(parameters[j], "sAinv") == 0 ) && (j < num_parameters-1) ) {
ssAinv=atoi(parameters[j+1]);
}
if ((strcmp(parameters[j], "sPsi") == 0 ) && (j < num_parameters-1) ) {
ssPsi=atoi(parameters[j+1]);
}
if ((strcmp(parameters[j], "nsHuff") == 0 ) && (j < num_parameters-1) ) {
nsHuff=atoi(parameters[j+1]);
nsHuff *=1024;
}
if ((strcmp(parameters[j], "psiSF") == 0 ) && (j < num_parameters-1) ) {
psiSearchFactor=atoi(parameters[j+1]);
}
j++;
}
free_parameters(num_parameters, ¶meters);
}
myicsa->T_A = ssA;
myicsa->T_AInv = ssAinv;
myicsa->T_Psi = ssPsi;
myicsa->tempNSHUFF = nsHuff;
myicsa->psiSearchFactorJump = psiSearchFactor * ssPsi;
printf("\n\t parameters for iCSA: sampleA=%d, sampleAinv=%d, samplePsi=%d",ssA,ssAinv,ssPsi);
printf("\n\t : nsHuff=%d, psiSearchFactor = %d --> jump = %d", nsHuff,psiSearchFactor, myicsa->psiSearchFactorJump);
}
int setDefaultSearchOptions_il (void *ail, char *search_options){
t_il *il = (t_il *) ail;
//il->defs.defaultFsearch = NULL;
il->defs.defaultIntersect2 = NULL;
il->defs.defaultIntersectN = NULL;
/** processing the parameters */
char delimiters[] = " =;";
int j,num_parameters;
char ** parameters;
if (search_options != NULL) {
parse_parameters(search_options,&num_parameters, ¶meters, delimiters);
for (j=0; j<num_parameters;j++) {
//setting the default function to Intersect-2-lists
if ((strcmp(parameters[j], "int2") == 0 ) && (j < num_parameters-1) ) {
if (strcmp(parameters[j+1], "merge") == 0 )
il->defs.defaultIntersect2 = merge2;
j++;
}
//setting the default function to Intersect-N-lists
else if ((strcmp(parameters[j], "intn") == 0 ) && (j < num_parameters-1) ) {
if (strcmp(parameters[j+1], "merge") == 0 )
il->defs.defaultIntersectN = merge_N_SIMD;
j++;
}
}
free_parameters(num_parameters, ¶meters);
}
fprintf(stderr,"\n ** Call to setDefaultSearchOptions_il: \"%s\"\n", search_options);
return 0;
}
/**
* A function to create a new load balancer.
* @param[in] api The deltacloud_api structure representing the connection
* @param[in] name The name to give to the new load balancer
* @param[in] realm_id The realm ID to put the new load balancer in
* @param[in] protocol The protocol to load balance
* @param[in] balancer_port The port the load balancer listens on
* @param[in] instance_port The port the load balancer balances to
* @param[in] params An array of deltacloud_create_parameter structures that
* represent any optional parameters to pass into the
* create call
* @param[in] params_length An integer describing the length of the params
* array
* @returns 0 on success, -1 on error
*/
int deltacloud_create_loadbalancer(struct deltacloud_api *api, const char *name,
const char *realm_id, const char *protocol,
int balancer_port, int instance_port,
struct deltacloud_create_parameter *params,
int params_length)
{
struct deltacloud_create_parameter *internal_params;
int ret = -1;
int pos;
if (!valid_api(api) || !valid_arg(name) || !valid_arg(realm_id)
|| !valid_arg(protocol))
return -1;
if (balancer_port < 0 || balancer_port > 65536) {
invalid_argument_error("balancer_port must be between 0 and 65536");
return -1;
}
if (instance_port < 0 || instance_port > 65536) {
invalid_argument_error("instance_port must be between 0 and 65536");
return -1;
}
if (params_length < 0) {
invalid_argument_error("params_length must be >= 0");
return -1;
}
internal_params = calloc(params_length + 5,
sizeof(struct deltacloud_create_parameter));
if (internal_params == NULL) {
oom_error();
return -1;
}
pos = copy_parameters(internal_params, params, params_length);
if (pos < 0)
/* copy_parameters already set the error */
goto cleanup;
if (deltacloud_prepare_parameter(&internal_params[pos++], "name", name) < 0 ||
deltacloud_prepare_parameter(&internal_params[pos++], "realm_id", realm_id) < 0 ||
deltacloud_prepare_parameter(&internal_params[pos++], "listener_protocol", protocol) < 0)
/* deltacloud_prepare_parameter already set the error */
goto cleanup;
if (prepare_int_parameter(&internal_params[pos++], "listener_balancer_port",
balancer_port) < 0)
/* prepare_int_parameter already set the error */
goto cleanup;
if (prepare_int_parameter(&internal_params[pos++], "listener_instance_port",
instance_port) < 0)
/* prepare_int_parameter already set the error */
goto cleanup;
if (internal_create(api, "load_balancers", internal_params, pos, NULL,
NULL) < 0)
/* internal_create already set the error */
goto cleanup;
ret = 0;
cleanup:
free_parameters(internal_params, pos);
SAFE_FREE(internal_params);
return ret;
}
static int lb_register_unregister(struct deltacloud_api *api,
struct deltacloud_loadbalancer *balancer,
const char *instance_id,
struct deltacloud_create_parameter *params,
int params_length,
const char *link)
{
struct deltacloud_link *thislink;
struct deltacloud_create_parameter *internal_params;
char *href;
int ret = -1;
int pos;
int rc;
if (!valid_api(api) || !valid_arg(balancer) || !valid_arg(instance_id))
return -1;
if (params_length < 0) {
invalid_argument_error("params_length must be >= 0");
return -1;
}
thislink = api_find_link(api, "load_balancers");
if (thislink == NULL)
/* api_find_link set the error */
return -1;
internal_params = calloc(params_length + 1,
sizeof(struct deltacloud_create_parameter));
if (internal_params == NULL) {
oom_error();
return -1;
}
pos = copy_parameters(internal_params, params, params_length);
if (pos < 0)
/* copy_parameters already set the error */
goto cleanup;
if (deltacloud_prepare_parameter(&internal_params[pos++], "instance_id",
instance_id) < 0)
/* deltacloud_prepare_parameter already set the error */
goto cleanup;
if (asprintf(&href, "%s/%s/%s", thislink->href, balancer->id, link) < 0) {
oom_error();
goto cleanup;
}
rc = internal_post(api, href, internal_params, pos, NULL, NULL);
SAFE_FREE(href);
if (rc < 0)
/* internal_post already set the error */
goto cleanup;
ret = 0;
cleanup:
free_parameters(internal_params, pos);
SAFE_FREE(internal_params);
return ret;
}
sds request(T4C* t4c, METHOD method, sds endPoint, Parameters* paramsArgument) {
sds result;
Parameters* oauthParams = new_parameters();
genOAuthParams(t4c, oauthParams);
Parameters* params = new_parameters();
buildParams(params, oauthParams, paramsArgument);
sds url = sdscatprintf(sdsempty(), "%s%s", baseUrl, endPoint);
sds oauthSignature = signature(t4c->consumerSecret, t4c->accessTokenSecret, method, url, params);
sds encodedSignature = url_encode(oauthSignature);
add_parameter(oauthParams, sdsnew("oauth_signature"), encodedSignature);
add_parameter(params, sdsnew("oauth_signature"), encodedSignature);
sds authorizeChild = join_parameters(oauthParams, ",");
sds authorize = sdscatprintf(sdsempty(), "Authorization: OAuth %s", authorizeChild);
sds path = join_parameters(params, "&");
if (DEBUG) {
printf("----------------------------\n");
printf("URL: %s\n", url);
printf("endPoint: %s\n", endPoint);
printf("path: %s\n", path);
printf("authorize: %s\n", authorize);
printf("----------------------------\n");
}
CURL* curl;
curl = curl_easy_init();
sds reqURL;
if (method == GET) {
reqURL = sdscatprintf(sdsempty(), "%s?%s", url, path);
curl_easy_setopt(curl, CURLOPT_URL, reqURL);
} else if (method == POST) {
curl_easy_setopt(curl, CURLOPT_POST, 1);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, path);
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, sdslen(path));
curl_easy_setopt(curl, CURLOPT_URL, url);
}
struct curl_slist *headers = NULL;
headers = curl_slist_append(headers, authorize);
curl_easy_setopt(curl, CURLOPT_HEADER, headers);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_result_stringlize_callback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&result);
curl_easy_perform(curl);
curl_easy_cleanup(curl);
sdsfree(oauthSignature);
sdsfree(encodedSignature);
sdsfree(reqURL);
sdsfree(url);
sdsfree(path);
sdsfree(authorize);
sdsfree(authorizeChild);
free_parameters(oauthParams);
return result;
}
void pbs_test(char *data) {
pbs_parameters pbs;
pbs_pk pk;
pbs_sk sk;
pbs_bank_state bstate;
pbs_client_state cstate;
pbs_signature signature;
pbs_workspace workspace;
struct timespec bank_start, bank_end, verify_start, verify_end;
long bank_nanos = 0;
long verify_nanos = 0;
int success, i = 0;
load_parameters(&pbs);
/* use this to generate keys */
/*
gen_keys(&pbs, &sk, &pk);
printf("skx:%s\n", BN_bn2hex(sk.x));
printf("pky:%s\n", BN_bn2hex(pk.y));
*/
load_keys(&sk, &pk);
int date1 = time(NULL) / 86400;
int date2 = date1 + 7;
int date3 = date1 + 28;
int infolen = 3 * sizeof(date1) + 6; /* 3 ints, 2 commas, null byte */
char info[infolen];
snprintf(info, infolen, "%d,%d,%d", date1, date2, date3);
/* printf("%s\n", info); */
/* do a bunch of signatures and verifies */
/* FIXME this can be made much more efficient */
char *pos = data;
for (i = 0; i < NUM_LOOPS_PER_RUN; ++i) {
BIGNUM *a = BN_new();
BIGNUM *b = BN_new();
BIGNUM *e = BN_new();
BIGNUM *r = BN_new();
BIGNUM *c = BN_new();
BIGNUM *s = BN_new();
BIGNUM *d = BN_new();
/* client sends bank request, bank sends back a,b,info */
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &bank_start);
bank_sign_init(a, b, &bstate, info, &pbs);
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &bank_end);
bank_nanos = get_timer_nanos(&bank_start, &bank_end);
/* client initialization */
sign_init(&cstate, &signature, &workspace);
/* client uses a,b,info and its message to produce e for bank */
sign_update(e, &cstate, &pbs, &pk, data, info, a, b);
/* bank uses e to produce r,c,s,d for client */
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &bank_start);
bank_sign_update(r, c, s, d, &bstate, &pbs, &sk, e);
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &bank_end);
bank_nanos += get_timer_nanos(&bank_start, &bank_end);
/* client finishes signature */
sign_final(&signature, r, c, s, d, &cstate, &pbs);
/* now verify */
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &verify_start);
success = verify(&signature, &pk, &pbs, info, data, &workspace);
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &verify_end);
verify_nanos = get_timer_nanos(&verify_start, &verify_end);
if (success != 0) {
printf("Signature incorrect\n");
}
pos += CELL_NETWORK_SIZE + 1;
BN_free(a);
BN_free(b);
BN_free(e);
BN_free(r);
BN_free(c);
BN_free(s);
BN_free(d);
free_bank_state(&bstate);
free_client_state(&cstate);
free_signature(&signature);
free_workspace(&workspace);
printf("pbs Bank effort: 1 signature %ld nanoseconds (%d/%d)\n", bank_nanos, i+1, NUM_LOOPS_PER_RUN);
printf("pbs Signature verification: 1 verify %ld nanoseconds (%d/%d)\n", verify_nanos, i+1, NUM_LOOPS_PER_RUN);
}
free_keys(&sk, &pk);
free_parameters(&pbs);
#ifdef DEBUG
/* signature results */
printBN(signature.delta, "delta:");
printBN(signature.sigma, "sigma:");
printBN(signature.omega, "omega:");
printBN(signature.rho, "rho:");
printf("info:%s\n", info);
printf("message:%s\n", message);
printBN(a, "a:");
printBN(b, "b:");
printBN(e, "e:");
printBN(r, "r:");
printBN(c, "c:");
printBN(s, "s:");
printBN(d, "d:");
printBN(cstate.t1, "t1:");
printBN(cstate.t2, "t2:");
printBN(cstate.t3, "t3:");
printBN(cstate.t4, "t4:");
printBN(cstate.epsilon, "epsilon:");
printBN(bstate.d, "d:");
printBN(bstate.s, "s:");
printBN(bstate.u, "u:");
printBN(pbs.g, "g:");
printBN(pbs.p, "p:");
printBN(pbs.q, "q:");
printBN(pk.y, "y:");
printBN(sk.x, "x:");
/* sanity checks */
BIGNUM *temp1 = BN_new();
BIGNUM *temp2 = BN_new();
BN_CTX *ctx = BN_CTX_new();
/* q |? p-1 */
BN_sub(temp1, pbs.p, BN_value_one());
BN_mod(temp2, temp1, pbs.q, ctx);
printf("q|p-1 remainder: %s\n", BN_bn2hex(temp2));
/* g^q =? 1 mod p */
BN_mod_exp(temp1, pbs.g, pbs.q, pbs.p, ctx);
printf("g^q =? 1 mod p result: %s\n", BN_bn2hex(temp1));
#endif
/*
BN_free(a);
BN_free(b);
BN_free(e);
BN_free(r);
BN_free(c);
BN_free(s);
BN_free(d);
free_bank_state(&bstate);
free_client_state(&cstate);
free_signature(&signature);
free_keys(&sk, &pk);
free_parameters(&pbs);
*/
}
int build_il (uint *source, uint length, char *build_options, void **ail) {
printf("\n CALL TO BUILD_IL !!\n");
t_il *il;
il = (t_il *) malloc (sizeof (t_il) * 1);
*ail = il;
/** processing the parameters */
char delimiters[] = " =;";
char filename[256] = "unnamed";
int j,num_parameters;
char ** parameters;
il->path2lzmaEncoder[0]= '\0';
il->minbcssize = MINBCSSIZE;
if (build_options != NULL) {
parse_parameters(build_options,&num_parameters, ¶meters, delimiters);
for (j=0; j<num_parameters;j++) {
if ((strcmp(parameters[j], "filename") == 0 ) && (j < num_parameters-1) ) {
strcpy(filename,parameters[j+1]);
j++;
}
else if ((strcmp(parameters[j], "minbcssize") == 0 ) && (j < num_parameters-1) ) {
il->minbcssize = atoi (parameters[j+1]);
j++;
}
else if ((strcmp(parameters[j], "path2lzmaencoder") == 0 ) && (j < num_parameters-1) ) {
strcpy(il->path2lzmaEncoder, (parameters[j+1]));
j++;
}
}
/*
if (il->path2lzmaEncoder[0] == '\0') {
printf("\n path2lzmaencoder NOT PROVIDED... Cannot continue!!"); exit(0);
}
*/
free_parameters(num_parameters, ¶meters);
}
printf("\n parameters of method: filename = %s, path2lzmaencoder=%s, minbcssize = %d\n", filename,il->path2lzmaEncoder, il->minbcssize);
{ /** 4 ** creates a temporal list of occurrences of each word (block-oriented).
gives also the len of each list */
uint i;
uint **occList;
uint *lenList;
uint maxPostValue;
fprintf(stderr,"\n paso 1 ");fflush(stderr);
parseAllPostingList(&(il->nlists), &lenList, &occList, &maxPostValue, source,length);
free(source);
fprintf(stderr,"\n paso 2 ");fflush(stderr);
il->lenList = lenList;
il->maxPostingValue = maxPostValue;
printf("numOflists = %u, maxPost = %u, \n", il->nlists, maxPostValue);
//calculates an stimation of the uncompressed representation of the index (using uints per posting)
il->sizeUncompressed = sizeof(uint) * (il->nlists) + //sizeoflenList
sizeof(uint*) * (il->nlists); //pointers to each posting (occs[i])
for (i=0; i< il->nlists;i++) { il->sizeUncompressed += (lenList[i] * sizeof(uint));}
fprintf(stderr,"stimated mem-usage of uncompressed posting-lists = %u bytes\n", il->sizeUncompressed);
/** 5 ** Encoding the occurrences **/
il->occs = (uint *) malloc (sizeof(uint) * (il->nlists+1));
//uint *numAbsSamplesperID = (uint *) malloc (sizeof(uint) * (il->nlists));
/** 5.b **
Computes the size of bc vector by simulating encoding process of gaps.
Sets also bitmaps for those terms indexed with a bitmap.
Computes and sets the values for "occs[]".
*/
uint bcssize=0;
{
uint nwords = il->nlists;
uint id,j;
uint prevvalue, gap, numbytes;
uint len, sampleP;
uint counterBCcodes =0;
uint counterBitmaps=0;
for (id=0; id<nwords; id++) {
len= il->lenList[id];
prevvalue= 0;
il->occs[id] = bcssize;
for (j=0; j<len; j++){
gap = occList[id][j]-prevvalue;
prevvalue = occList[id][j];
SIZE_DEC_TO_BC(gap,numbytes);
bcssize += numbytes;
counterBCcodes++;
}
}
il->occs[nwords] = bcssize;
fprintf(stderr,"\n Number of gaps to store is = %d",counterBCcodes);
fprintf(stderr,"\n Size of bcs Vector should be %d bytes",bcssize);
}
il->bcsSize = bcssize;
il->bcs = (byte *) malloc (sizeof(byte) * il->bcsSize);
/** 5.c ** encodes gaps into bcs */
{
uint nwords = il->nlists;
byte *bcs = il->bcs;
uint id,j;
uint prevvalue, gap;
uint len;
uint bcpos=0;
for (id=0; id<nwords; id++) {
len= il->lenList[id];
prevvalue=0;
for (j=0; j<len; j++){
gap = occList[id][j]-prevvalue;
prevvalue = occList[id][j];
DEC_TO_BC(bcs, bcpos, gap); //bcpos is increased.
}
}
fprintf(stderr,"\n Gaps were encoded with bc ==> %d bytes",bcpos);
}
/** 6. Now the lists that compress much better by using vbyte+lzma
* instead of just vbyte, are compressed with vbyte+lzma
*/
/*
{ uint j,id;
uint nwords = il->nlists;
byte *newbcs = (byte *) malloc (sizeof(byte) * il->bcsSize);
uint *newoccs = (uint *) malloc (sizeof (uint) * (nwords+1));
il->useslzma = (uint *) malloc (sizeof(uint) * ((nwords + W -1)/W));
initLzmaStruct (&il->lzmadata);
for (j=0;j< ((nwords + W -1)/W) ; j++) il->useslzma[j]= 0000;
uint bcpos=0;
int res=0;
uint lenbcs;
uint lenlzma;
for (id=0; id<nwords; id++) {
byte *inb; uint inn; //pointer to compressed file in RAM and its size.
newoccs[id]= bcpos;
lenbcs = il->occs[id+1]-il->occs[id];
byte *bcsdata = &(il->bcs[il->occs[id]]); // ** ptr to bytecoded data * /
if (lenbcs> il->minbcssize) // * CHECKS THE THRESHOLD FOR lenbcs TO CHECK IF LZMA COULD BE OK ** /
{
byte *bcsdata = &(il->bcs[il->occs[id]]); // ** ptr to bytecoded data * /
res = compressDataExt(bcsdata,lenbcs, il->path2lzmaEncoder);
if (!res) {
loadDataExt(&inb,&inn);
createLzmaExt(inb, inn, il->lzmadata);
lenlzma = sizeBytesLzmaStruct(il->lzmadata); // ** memory need if lzma is used * /
printf("\n [id= %6d/%6d] lenbcs = %8d, lenbcs+lzma = %8d ==> usesLzma = %d (f/t?) (bcpos=%d)",id, nwords,lenbcs,lenlzma, (lenlzma<lenbcs),bcpos);
if (lenlzma < lenbcs) {
writeBuffLzmaStruct(newbcs,bcpos,il->lzmadata);
bcpos +=lenlzma;
bitset(il->useslzma, id);
}
else {
memcpy( (void *) &(newbcs[bcpos]), (const void *) bcsdata, lenbcs);
bcpos +=lenbcs;
}
free(inb);
}
else {
printf("\n LZMA compression failed!. Aborting... \n");exit(0);
}
}
else {
memcpy( (void *) &(newbcs[bcpos]), (const void *) bcsdata, lenbcs);
bcpos +=lenbcs;
}
}// * for * /
newoccs[nwords]=bcpos;
free(il->bcs);
free(il->occs);
il->bcs = newbcs;
il->bcsSize = bcpos;
il->occs = newoccs;
}
}
*/
{ uint j,id;
uint nwords = il->nlists;
// byte *newbcs = (byte *) malloc (sizeof(byte) * il->bcsSize);
// uint *newoccs = (uint *) malloc (sizeof (uint) * (nwords+1));
uint bcpos=0;
int res=0;
uint lenbcs;
uint lenlzma;
tvbytelzma *data = generateData(il,nwords);
il->useslzma=NULL;
OCCLISTCHILD =occList;
res = compressedParallelFather(il,data, nwords, il->minbcssize);
if (!res) {
printf("\n compression in parallel worked fine!\n");
}
else {
printf("\n compression in parallel failed!\n");
}
initLzmaStruct (&il->lzmadata);
byte *newbcs = (byte *) malloc (sizeof(byte) * il->bcsSize);
uint *newoccs = (uint *) malloc (sizeof (uint) * (nwords+1));
il->useslzma = (uint *) malloc (sizeof(uint) * ((nwords + W -1)/W));
for (j=0;j< ((nwords + W -1)/W) ; j++) il->useslzma[j]= 0000;
for (id=0; id<nwords; id++) {
newoccs[id]= bcpos;
lenbcs = il->occs[id+1]-il->occs[id];
byte *bcsdata = &(il->bcs[il->occs[id]]); // ** ptr to bytecoded data * /
if (lenbcs != data[id].ilen) { printf("\n ilen does not coincide for term %lu", (ulong)id); exit(0);}
if (lenbcs> il->minbcssize) /** CHECKS THE THRESHOLD FOR lenbcs TO CHECK IF LZMA COULD BE OK **/
{
lenbcs = data[id].ilen;
lenlzma = data[id].olen;
printf("\n [id= %6d/%6d] lenbcs = %8d, lenbcs+lzma = %8d ==> usesLzma = %d (f/t?) (bcpos=%d)",id, nwords,lenbcs,lenlzma, (lenlzma<lenbcs),bcpos);
if (lenlzma < lenbcs) {
memcpy( (void *) &(newbcs[bcpos]), (const void *) data[id].outputdata, lenlzma);
bcpos +=lenlzma;
bitset(il->useslzma, id);
}
else {
if (lenbcs != lenlzma) {printf("\n LENBCS != LENLZMA y deberían ser iguales\n"); exit(0);}
memcpy( (void *) &(newbcs[bcpos]), (const void *) bcsdata, lenbcs);
bcpos +=lenbcs;
}
//free(data[id].outputdata);
}
else {
memcpy( (void *) &(newbcs[bcpos]), (const void *) bcsdata, lenbcs);
bcpos +=lenbcs;
}
free(data[id].outputdata);
}/* for */
newoccs[nwords]=bcpos;
//for (i=0;i< nwords; i++) free(data[i].outputdata);
free(data);
free(il->bcs);
free(il->occs);
il->bcs = newbcs;
il->bcsSize = bcpos;
il->occs = newoccs;
}
/** checking that the lists are decoded successfully *************/
fprintf(stderr,"\n ! CHECKING that the decoded lists are identical to the original ones. ");
{ //checking:: decoding a list
uint i, id ,len;
uint *list;
for (id=0;id < il->nlists;id++) {
//if ( shouldUseBitmap(il,id) ) continue;
extractList_il (*ail, id, &list, &len);
//extractList (void *ail, uint id, uint **list, uint *len)
for (i=0;i<len;i++){
if (list[i] != (occList[id][i]-DOCid_ADD) ) {
fprintf(stderr,"\n decoding of lists failed for id = %d: DIFFERENT!!: (%d,%d)",id, list[i],occList[id][i]);
exit(0);
}
}
free(list);
}
}
fprintf(stderr,"\n ! Decoding the list of occurrences worked fine (the same as previous values) ");
fprintf(stderr,"\n ! CHECKING that the decoded lists are identical to the original ones (extract_no_malloc). ");
{ //checking:: decoding a list
uint i, id ,len;
uint *list = (uint *) malloc(sizeof(uint) * (il->maxPostingValue+1));
for (id=0;id < il->nlists;id++) {
//if ( shouldUseBitmap(il,id) ) continue;
//extractList_il (*ail, id, &list, &len);
extractListNoMalloc_il(*ail,id,list,&len);
//extractList (void *ail, uint id, uint **list, uint *len)
for (i=0;i<len;i++){
if (list[i] != (occList[id][i]-DOCid_ADD)) {
//fprintf(stderr,"\n decoding of lists failed for id = %d: DIFFERENT!!",id);
fprintf(stderr,"\n decoding of lists failed for id = %d: DIFFERENT!!: (%d,%d)",id, list[i],occList[id][i]);
exit(0);
}
}
}
free(list);
}
fprintf(stderr,"\n ! Decoding the list of occurrences worked fine (the same as previous values) ");
/******************************************************/
//frees memory
for (i=0;i<il->nlists;i++) free(occList[i]);
free(occList);
}
fprintf(stderr,"\n The index has already been built!!\n");
return 0;
}
int build_il (uint *source, uint length, char *build_options, void **ail) {
printf("\n CALL TO BUILD_IL !!\n");
t_il *il;
il = (t_il *) malloc (sizeof (t_il) * 1);
*ail = il;
/** processing the parameters */
char delimiters[] = " =;";
char filename[256] = "unnamed";
int j,num_parameters;
char ** parameters;
uint pfdThreshold= DEFAULT_PFD_THRESHOLD;
if (build_options != NULL) {
parse_parameters(build_options,&num_parameters, ¶meters, delimiters);
for (j=0; j<num_parameters;j++) {
if ((strcmp(parameters[j], "filename") == 0 ) && (j < num_parameters-1) ) {
strcpy(filename,parameters[j+1]);
j++;
}
else if ((strcmp(parameters[j], "pfdThreshold") == 0 ) && (j < num_parameters-1) ) {
pfdThreshold=atoi(parameters[j+1]);
j++;
}
}
free_parameters(num_parameters, ¶meters);
}
printf("\n parameters of method: pfdThreshold = %u, filename = %s\n", pfdThreshold, filename);
il->pfdThreshold = pfdThreshold;
{ /** 4 ** creates a temporal list of occurrences of each word (block-oriented).
gives also the len of each list */
uint i;
uint **occList;
uint *lenList;
uint maxPostValue;
// //allocating memory.
// uint **occs;
// occs = (uint **) malloc (sizeof(uint *) * n);
// for (i=0;i<n;i++) occs[i]= (uint *) malloc (sizeof(uint) * wcsa->freqs[i]);
fprintf(stderr,"\n paso 1 ");fflush(stderr);
parseAllPostingList(&(il->nlists), &lenList, &occList, &maxPostValue, source,length);
free(source);
fprintf(stderr,"\n paso 2 ");fflush(stderr);
il->lenList = lenList;
il->maxPostingValue = maxPostValue;
printf("numOflists = %ld, maxPost = %ld, \n",
(long)il->nlists, (long)maxPostValue);
//calculates an stimation of the uncompressed representation of the index (using uints per posting)
il->sizeUncompressed = sizeof(uint) * (il->nlists) + //sizeoflenList
sizeof(uint*) * (il->nlists); //pointers to each posting (occs[i])
for (i=0; i< il->nlists;i++) { il->sizeUncompressed += (lenList[i] * sizeof(uint));}
fprintf(stderr,"estimated mem-usage of uncompressed posting-lists = %ld bytes\n", (long)il->sizeUncompressed);
/** 5 ** Encoding the occurrences **/
il->occs = (uint *) malloc (sizeof(uint) * (il->nlists+1));
il->clen = (uint *) malloc (sizeof(uint) * (il->nlists));
uint maxlenlist=0;
for (i=0; i< il->nlists;i++) { if (maxlenlist<lenList[i]) maxlenlist=lenList[i];}
fprintf(stderr,"\n maxlenlist = %lu",(ulong)maxlenlist);
uint *DIFF= (uint *) malloc (sizeof(uint)*(maxlenlist+(QMX_BS2*4))); //stores diferences.
uint *C= (uint *) malloc (sizeof(uint)*(maxlenlist+(QMX_BS2*4))); //compressed data.
{ //counts the number of terms indexed with bitmaps and reserves memory for them.
uint numPFD=0;
uint nwords = il->nlists;
for (i=0;i<nwords; i++){
//if (shouldUsePFD(il, i))
numPFD++;
}
fprintf(stderr,"\n Number of lists = %u, %u using PFD, %d using variableLenght codes",il->nlists, numPFD, il->nlists - numPFD);
il->numPFD = numPFD;
}
/** 5.b **
Computes the size of bc vector by simulating encoding process of gaps.
Sets also bitmaps for those terms indexed with a bitmap.
Computes and sets the values for "occs[]".
*/
uint bcssize=0;
uint pfdsize=0;
il->padUints = 0;
{
uint nwords = il->nlists;
uint id,j;
uint prevvalue, gap;
uint len;
uint counterBCcodes =0;
uint counterPFC =0;
for (id=0; id<nwords; id++) {
len= il->lenList[id];
/*
if (! shouldUsePFD(il,id) ) { // ** spire **
prevvalue= 0;
uint numbytes;
il->occs[id] = bcssize;
for (j=0; j<len; j++){
gap = occList[id][j]-prevvalue;
prevvalue = occList[id][j];
SIZE_DEC_TO_BC(gap,numbytes);
bcssize += numbytes;
counterBCcodes++;
}
}
else */
{
//Calculates DIFF.
prevvalue= 0;
il->occs[id] = pfdsize;
for (j=0; j<len; j++){
gap = occList[id][j]-prevvalue;
DIFF[j]=gap;
prevvalue = occList[id][j];
counterPFC++;
}
//size of pfd compressed data for current list "id"
int N = QMX_Compression(DIFF, (int) len, C);
il->clen[id]=N;
pfdsize+= N;
//padding info "to aling to 8 ints = 256bits"
pfdsize+= (8-N%8);
il->padUints += (8-N%8);
}
}
il->occs[nwords]=pfdsize;
fprintf(stderr,"\n Number of gaps to store with BCsis = %lu", (ulong)counterBCcodes);
fprintf(stderr,"\n Number of gaps to store with PFDelta = %lu",(ulong)counterPFC);
fprintf(stderr,"\n Size of bcs Vector should be %lu bytes",(ulong)bcssize);
fprintf(stderr,"\n Size of PFD Data should be %lu bytes (%u uints)",(ulong)pfdsize*sizeof(uint), pfdsize);
}
il->bcsSize = bcssize;
il->bcs = (byte *) malloc (sizeof(byte) * il->bcsSize);
il->cdataSize = pfdsize;
il->cdata = (uint *)malloc (sizeof(uint) * il->cdataSize + sizeof(uint)*(QMX_BS2*4) +200000000);
il->cdata[il->cdataSize -1] = 0000;
il->cdata[il->cdataSize +0] = 0000;
il->cdata[il->cdataSize +1] = 0000;
il->cdata[il->cdataSize +2] = 0000;
/** 5.c ** encodes gaps into bcs or cdata for pfd */
{
uint nwords = il->nlists;
uint *cdata = il->cdata;
uint id,j;
uint prevvalue, gap;
uint len;
uint bcpos=0;
for (id=0; id<nwords; id++) {
len= il->lenList[id];
prevvalue=0;
/*if ( ! shouldUsePFD(il,id) ) {
byte *bcs = il->bcs;
for (j=0; j<len; j++){
gap = occList[id][j]-prevvalue;
prevvalue = occList[id][j];
DEC_TO_BC(bcs, bcpos, gap); //bcpos is increased.
}
}
else */
{
//Calculates DIFF.
prevvalue= 0;
for (j=0; j<len; j++){
gap = occList[id][j]-prevvalue;
DIFF[j]=gap;
prevvalue = occList[id][j];
}
int N = QMX_Compression(DIFF, (int) len, cdata);
//adds padding
cdata = cdata + N; // + (8-N%8);
for (j=0;j< ((uint)(8-N%8));j++) {
*cdata=0;
cdata++;
}
}
}
fprintf(stderr,"\n Gaps encoded with bc ==> %u bytes",bcpos);
fprintf(stderr,"\n Gaps encoded with pfd ==> %lu bytes :: %lu uints (%lu)", (ulong)(sizeof(uint)*(cdata- il->cdata)),(ulong) il->cdataSize, (ulong) (cdata - il->cdata));
}
free(DIFF);
free(C);
/** checking that the lists are decoded successfully *************/
fprintf(stderr,"\n ! CHECKING that the decoded lists are identical to the original ones. ");
{ //checking:: decoding a list
uint i, id ,len;
uint *list;
for (id=0;id < il->nlists;id++) {
extractList_il (*ail, id, &list, &len);
for (i=0;i<len;i++){
if (list[i] != (occList[id][i]-DOCid_ADD) ) {
fprintf(stderr,"\n decoding of lists failed for id = %d: DIFFERENT [i=%u,len=%u]!!: (%d,%d)",id, i, len, list[i],occList[id][i]);
exit(0);
}
}
free(list);
}
}
fprintf(stderr,"\n ! Decoding the list of occurrences worked fine (the same as previous values) ");
fprintf(stderr,"\n ! CHECKING that the decoded lists are identical to the original ones (extract_no_malloc). ");
{ //checking:: decoding a list
uint i, id ,len;
uint *list = (uint *) malloc(sizeof(uint) * (il->maxPostingValue+1+ QMX_BS2));
for (id=0;id < il->nlists;id++) {
extractListNoMalloc_il(*ail,id,list,&len);
for (i=0;i<len;i++){
if (list[i] != (occList[id][i]-DOCid_ADD)) {
fprintf(stderr,"\n decoding of lists failed for id = %d: DIFFERENT!!: (%d,%d)",id, list[i],occList[id][i]);
exit(0);
}
}
}
free(list);
}
fprintf(stderr,"\n ! Decoding the list of occurrences worked fine (the same as previous values) ");
/******************************************************/
//frees memory
for (i=0;i<il->nlists;i++) free(occList[i]);
free(occList);
}
fprintf(stderr,"\n The index has already been built!!\n");
return 0;
}
/*////////////////////
//Building the Index//
////////////////////*/
int build_index(uchar *text, ulong length, char *build_options, void **index) {
/*if (text[length-1]!='\0') return 2;*/
ulong i, *p, *sa_diff;
long overshoot;
TSA_Un *_index= (TSA_Un *) malloc(sizeof(TSA_Un));
uchar *x;
FILE *f;
char fnamext[1024];
char fnameaux[1024];
char delimiters[] = " =;";
int j,num_parameters;
char ** parameters;
int copy_text=false; /* don't copy text by default */
int free_text=false; /* don't free text by default */
int withload=false; /* don't load SA and BPE by default */
int samplerate=64; /* samplerate for bpe */
int max_phrase=256;
bool verbose=false;
double cutoff=100.0;
bool SA_treap=true,SA_psi=false;
if (!_index) return 1;
if (build_options != NULL) {
parse_parameters(build_options,&num_parameters, ¶meters, delimiters);
for (j=0; j<num_parameters; j++) {
if (strcmp(parameters[j], "copy_text") == 0 )
copy_text=true;
else if (strcmp(parameters[j], "withload") == 0 )
withload=true;
else if (strcmp(parameters[j], "filename") == 0 ) {
strcpy(fnamext,parameters[j+1]);
j++;
} else if ((strcmp(parameters[j], "samplerate") == 0 ) && (j < num_parameters-1) ) {
samplerate=atoi(parameters[j+1]);
j++;
} else if ((strcmp(parameters[j], "max_phrase") == 0 ) && (j < num_parameters-1) ) {
max_phrase=atoi(parameters[j+1]);
j++;
} else if ((strcmp(parameters[j], "cutoff") == 0 ) && (j < num_parameters-1) ) {
cutoff=atof(parameters[j+1]);
j++;
} else if (strcmp(parameters[j], "free_text") == 0 )
free_text=true;
else if (strcmp(parameters[j], "verbose") == 0 )
verbose=true;
else if (strcmp(parameters[j], "SA_treap") == 0 ) {
SA_treap=true;
SA_psi=false;
} else if (strcmp(parameters[j], "SA_psi") == 0 ) {
SA_treap=false;
SA_psi=true;
}
}
free_parameters(num_parameters, ¶meters);
}
//printf("samplerate = %lu\n",samplerate);
/* Consistence of parameters */
if ((!copy_text) && (free_text))
return 5;
/* */
if ( !copy_text ) {
_index->text = text;
_index->own=false;
}
else {
_index->text = (uchar *) malloc(sizeof(uchar)*length);
if (!_index->text) return 1;
for (i=0; i<length; i++) _index->text[i]=text[i];
_index->own=true;
}
if ( free_text )
free(text);
_index->n=length;
/* Make suffix array */
if (withload) {
ulong filename_len;
p= (ulong *) malloc (sizeof(ulong)*(length));
if (!p) return 1;
sprintf (fnameaux,"%s.sa",fnamext);
f = fopen (fnameaux,"r");
if (fread (&filename_len,sizeof(ulong),1,f) != 1) return 25;
assert(filename_len==_index->n);
if (fread (p,sizeof(ulong),filename_len,f) != filename_len) return 25;
if (fclose(f) != 0) return 28;
} else {
overshoot = init_ds_ssort(500, 2000);
p= (ulong *) malloc (sizeof(ulong)*(length));
if (!p) return 1;
x= (uchar *) malloc (sizeof(uchar)*(length+overshoot));
if (!x) return 1;
for (i=0; i<length; i++) x[i]=_index->text[i];
ds_ssort( x, p, _index->n);
free(x);
}
/* Make bpe */
if (withload && false ) {
int error;
sprintf (fnameaux,"%s.bpe",fnamext);
f = fopen (fnameaux,"r");
_index->bpe = new BPE(f,&error);
if (error !=0) return error;
if (fclose(f) != 0) return 28;
} else {
if (SA_treap) {
sa_diff= (ulong *) malloc (sizeof(ulong)*(length+3));
if (!sa_diff) return 1;
for (i=0; i<length-1; i++) {
assert(p[i+1]-p[i]+length>0);
sa_diff[i+1]=p[i+1]-p[i]+length;
}
free(p);
ulong maximo=0;
for (i=0; i<length-1; i++) {
if (maximo < sa_diff[i+1]) maximo=sa_diff[i+1];
}
sa_diff[0]=maximo+1;
sa_diff[length+1]=maximo+2;
sa_diff[length+2]=maximo+3;
_index->bpe = new BPE(sa_diff,length-1+3, max_phrase, cutoff, verbose);
}
if (SA_psi) {
ulong *ip= (ulong *) malloc (sizeof(ulong)*(length));
for (i=0; i<length; i++) ip[p[i]] = i;
for (i=0; i<length; i++) assert(ip[p[i]] == i);
ulong *Psi= (ulong *) malloc (sizeof(ulong)*(length));
for (i=0; i<length; i++) if (p[i] == length-1) Psi[i] = ip[0];
else Psi[i] = ip[p[i]+1];
ulong ini=ip[0];
free(ip);
sa_diff= (ulong *) malloc (sizeof(ulong)*length);
if (!sa_diff) return 1;
for (i=0; i<length-1; i++) {
assert(p[i+1]-p[i]+length>0);
sa_diff[i]=p[i+1]-p[i]+length;
}
free(p);
_index->bpe = new BPE(sa_diff,Psi,ini,length-1,verbose);
}
}
/* Make suffix array again */
if (withload) {
ulong filename_len;
p= (ulong *) malloc (sizeof(ulong)*(length));
if (!p) return 1;
sprintf (fnameaux,"%s.sa",fnamext);
f = fopen (fnameaux,"r");
if (fread (&filename_len,sizeof(ulong),1,f) != 1) return 25;
assert(filename_len==_index->n);
if (fread (p,sizeof(ulong),filename_len,f) != filename_len) return 25;
if (fclose(f) != 0) return 28;
} else {
overshoot = init_ds_ssort(500, 2000);
p= (ulong *) malloc (sizeof(ulong)*(length));
if (!p) return 1;
x= (uchar *) malloc (sizeof(uchar)*(length+overshoot));
if (!x) return 1;
for (i=0; i<length; i++) x[i]=_index->text[i];
ds_ssort( x, p, _index->n);
free(x);
}
/*
////////////////////////////////////////////////////
sa_diff= (ulong *) malloc (sizeof(ulong)*length);
for (i=0;i<length-1;i++){
assert(p[i+1]-p[i]+length>0);
sa_diff[i]=p[i+1]-p[i]+length;
}
ulong *z2;
z2=_index->bpe->dispairall();
printf("Check SA_diff todo\n");
for (i=0;i<length-1;i++){
if (z2[i]-sa_diff[i] !=0) {printf("%lu, %lu %lu,%lu\n",i, z2[i]-sa_diff[i],z2[i],sa_diff[i]);fflush(stdout);}
}
printf("End Check SA_diff todo\n");
free(z2);
printf("End Check SA_diff 2\n");
for (ulong mmm=1; mmm < 5000; mmm++) {
printf("Check SA_diff %lu ",mmm);
for (i=0;i<length-1-(mmm-1);i++){
z2=_index->bpe->dispair(i,mmm);
//if (i % (n/10) == 0) {printf("C2 %lu\n",i);fflush(stdout);}
for (ulong mm =1 ; mm <= mmm; mm++)
if (z2[mm]-sa_diff[i+mm-1] !=0) {printf("T%lu %lu, %lu %lu,%lu\n",mm,i, z2[mm]-sa_diff[i+mm-1],z2[mm],sa_diff[i+mm-1]);fflush(stdout);}
free(z2);
}
printf("End Check SA_diff %lu\n",mmm);fflush(stdout);
}
free(sa_diff);
/////////////////////////////////////////////////////////
*/
/* Make samplerate */
_index->samplerate = samplerate;
_index->ns = (length-1)/samplerate+1;
if (((length-1) % samplerate) != 0) _index->ns++;
_index->pos = (ulong *) malloc (sizeof(ulong)*_index->ns);
//_index->pos[0]=p[0];
j=0;
for (i=0; i < length ; i+=samplerate) {
if (i != length-1) {
//if (p[_index->bpe->BR->prev(i)] != _index->pos[j-1]) {
_index->pos[j]=p[_index->bpe->BR->prev(i)];
j++;
// }
} else {
_index->pos[j]=p[i];
j++;
}
}
if (((length-1) % samplerate) != 0) _index->pos[j]=p[length-1];
_index->ns=j+1;
/*
_index->samplerate = samplerate;
_index->ns = (length-1)/samplerate+1;
if (((length-1) % samplerate) != 0) _index->ns++;
_index->pos = (ulong *) malloc (sizeof(ulong)*_index->ns);
j=0;
for (i=0; i < length ; i+=samplerate) {
_index->pos[j]=p[i];
j++;
}
if (((length-1) % samplerate) != 0) _index->pos[j]=p[length-1];
assert(j+1==_index->ns);
*/
free(p);
(*index) = _index;
return 0;
}
/**
* A function to create a new storage snapshot.
* @param[in] api The deltacloud_api structure representing the connection
* @param[in] volume The volume to take the snapshot from
* @param[in] params An array of deltacloud_create_parameter structures that
* represent any optional parameters to pass into the
* create call
* @param[in] params_length An integer describing the length of the params
* array
* @param[out] snap_id The snapshot_id returned by the create call
* @returns 0 on success, -1 on error
*/
int deltacloud_create_storage_snapshot(struct deltacloud_api *api,
struct deltacloud_storage_volume *volume,
struct deltacloud_create_parameter *params,
int params_length,
char **snap_id)
{
struct deltacloud_create_parameter *internal_params;
struct deltacloud_storage_snapshot snap;
char *data = NULL;
int ret = -1;
int pos;
if (!valid_api(api) || !valid_arg(volume))
return -1;
if (params_length < 0) {
invalid_argument_error("params_length must be >= 0");
return -1;
}
internal_params = calloc(params_length + 1,
sizeof(struct deltacloud_create_parameter));
if (internal_params == NULL) {
oom_error();
return -1;
}
pos = copy_parameters(internal_params, params, params_length);
if (pos < 0)
/* copy_parameters already set the error */
goto cleanup;
if (deltacloud_prepare_parameter(&internal_params[pos++], "volume_id",
volume->id) < 0)
/* deltacloud_create_parameter already set the error */
goto cleanup;
if (internal_create(api, "storage_snapshots", internal_params, pos, &data,
NULL) < 0)
/* internal_create already set the error */
goto cleanup;
if (snap_id != NULL) {
if (internal_xml_parse(data, "storage_snapshot", parse_one_storage_snapshot,
1, &snap) < 0)
/* internal_xml_parse set the error */
goto cleanup;
*snap_id = strdup(snap.id);
deltacloud_free_storage_snapshot(&snap);
if (*snap_id == NULL) {
oom_error();
goto cleanup;
}
}
ret = 0;
cleanup:
free_parameters(internal_params, pos);
SAFE_FREE(internal_params);
SAFE_FREE(data);
return ret;
}
