/** Create a new PageInfo loading the information from a previously
* dumped PageInfo inside val.
*
* @return pointer to the new PageInfo or NULL if failure
*/
static PageInfo *
page_info_load(const MDB_val *val) {
PageInfo *pi = calloc(1, sizeof(*pi));
if (!pi)
return 0;
char *data = val->mv_data;
size_t i = 0;
size_t j;
char * d;
#define PAGE_INFO_READ(x) for (j=0, d=(char*)&(x); j<sizeof(x); d[j++] = data[i++])
unsigned short curl_size;
PAGE_INFO_READ(curl_size);
unsigned short url_size = 4*curl_size + 1;
int enough_memory = 0;
do {
if (!(pi->url = realloc(pi->url, url_size))) {
free(pi);
return 0;
}
int dec = smaz_decompress(data + i, curl_size, pi->url, url_size);
if ((unsigned int)dec < url_size) {
enough_memory = 1;
url_size = dec;
} else {
url_size *= 2;
}
} while (!enough_memory);
pi->url[url_size] = '\0';
i += curl_size;
PAGE_INFO_READ(pi->score);
PAGE_INFO_READ(pi->linked_from);
PAGE_INFO_READ(pi->depth);
PAGE_INFO_READ(pi->n_crawls);
if (pi->n_crawls > 0) {
PAGE_INFO_READ(pi->first_crawl);
if (pi->n_crawls > 1) {
PAGE_INFO_READ(pi->last_crawl);
PAGE_INFO_READ(pi->n_changes);
} else {
pi->last_crawl = pi->first_crawl;
}
PAGE_INFO_READ(pi->content_hash_length);
if (!(pi->content_hash = malloc(pi->content_hash_length))) {
free(pi->url);
free(pi);
return 0;
}
for (j=0; j<pi->content_hash_length; pi->content_hash[j++] = data[i++]);
}
return pi;
}
/**
* Decode the simple type "String"
*
* @param out The out parameter, returning a string contaning the decoded value
* @param bufferPos The buffer containing the BIXIE encoded data
*
* @return Returns and integer error flag if an error occurred
*/
int decodeString(char** out, byte** bufferPos) {
int sizeOfLeadingVarint = 1, inputSize;
char* inputSizeString;
char tempStr[4096];
while(((*bufferPos)[sizeOfLeadingVarint-1] & 0x80) == 0x80) {
sizeOfLeadingVarint++;
}
decodeInt(&inputSizeString, bufferPos);
inputSize = atoi(inputSizeString);
free(inputSizeString);
int outBytes = smaz_decompress(*bufferPos, inputSize, tempStr, sizeof(tempStr));
*out = (char*) malloc(outBytes *sizeof(char) + 1);
memcpy(*out, tempStr, outBytes);
(*out)[outBytes] = '\0';
*bufferPos += inputSize;
return NORMAL_RUN;
}
StringName PHashTranslation::get_message(const StringName& p_src_text) const {
int htsize = hash_table.size();
if (htsize==0)
return StringName();
CharString str = p_src_text.operator String().utf8();
uint32_t h = hash(0,str.get_data());
DVector<int>::Read htr = hash_table.read();
const uint32_t *htptr = (const uint32_t*)&htr[0];
DVector<int>::Read btr = bucket_table.read();
const uint32_t *btptr = (const uint32_t*)&btr[0];
DVector<uint8_t>::Read sr = strings.read();
const char *sptr= (const char*)&sr[0];
uint32_t p = htptr[ h % htsize];
//print_line("String: "+p_src_text.operator String());
//print_line("Hash: "+itos(p));
if (p==0xFFFFFFFF) {
// print_line("GETMSG: Nothing!");
return StringName(); //nothing
}
const Bucket &bucket = *(const Bucket*)&btptr[p];
h = hash(bucket.func,str.get_data());
int idx=-1;
for(int i=0;i<bucket.size;i++) {
if (bucket.elem[i].key==h) {
idx=i;
break;
}
}
//print_line("bucket pos: "+itos(idx));
if (idx==-1) {
// print_line("GETMSG: Not in Bucket!");
return StringName();
}
if (bucket.elem[idx].comp_size == bucket.elem[idx].uncomp_size) {
String rstr;
rstr.parse_utf8(&sptr[ bucket.elem[idx].str_offset ], bucket.elem[idx].uncomp_size );
// print_line("Uncompressed, size: "+itos(bucket.elem[idx].comp_size));
// print_line("Return: "+rstr);
return rstr;
} else {
CharString uncomp;
uncomp.resize( bucket.elem[idx].uncomp_size+1 );
smaz_decompress(&sptr[ bucket.elem[idx].str_offset ], bucket.elem[idx].comp_size,uncomp.ptr(),bucket.elem[idx].uncomp_size );
String rstr;
rstr.parse_utf8(uncomp.get_data());
// print_line("Compressed, size: "+itos(bucket.elem[idx].comp_size));
// print_line("Return: "+rstr);
return rstr;
}
}
int main(void) {
char in[512];
char out[4096];
char d[4096];
int comprlen, decomprlen;
int j, ranlen;
int times = 1000000;
char *strings[] = {
"This is a small string",
"foobar",
"the end",
"not-a-g00d-Exampl333",
"Smaz is a simple compression library",
"Nothing is more difficult, and therefore more precious, than to be able to decide",
"this is an example of what works very well with smaz",
"1000 numbers 2000 will 10 20 30 compress very little",
"and now a few italian sentences:",
"Nel mezzo del cammin di nostra vita, mi ritrovai in una selva oscura",
"Mi illumino di immenso",
"L'autore di questa libreria vive in Sicilia",
"try it against urls",
"http://google.com",
"http://programming.reddit.com",
"http://github.com/antirez/smaz/tree/master",
"/media/hdb1/music/Alben/The Bla",
NULL
};
j=0;
while(strings[j]) {
int comprlevel;
comprlen = smaz_compress(strings[j],strlen(strings[j]),out,sizeof(out));
comprlevel = 100-((100*comprlen)/strlen(strings[j]));
decomprlen = smaz_decompress(out,comprlen,d,sizeof(d));
if (strlen(strings[j]) != (unsigned)decomprlen ||
memcmp(strings[j],d,decomprlen))
{
printf("BUG: error compressing '%s'\n", strings[j]);
exit(1);
}
if (comprlevel < 0) {
printf("'%s' enlarged by %d%%\n",strings[j],-comprlevel);
} else {
printf("'%s' compressed by %d%%\n",strings[j],comprlevel);
}
j++;
}
printf("Encrypting and decrypting %d test strings...\n", times);
while(times--) {
char charset[]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvxyz/. ";
ranlen = random() % 512;
for (j = 0; j < ranlen; j++) {
if (times & 1)
in[j] = charset[random() % (sizeof(charset)-1)];
else
in[j] = (char)(random() & 0xff);
}
comprlen = smaz_compress(in,ranlen,out,sizeof(out));
decomprlen = smaz_decompress(out,comprlen,d,sizeof(out));
if (ranlen != decomprlen || memcmp(in,d,ranlen)) {
printf("Bug! TEST NOT PASSED\n");
exit(1);
}
/* printf("%d -> %d\n", comprlen, decomprlen); */
}
printf("TEST PASSED :)\n");
return 0;
}
