void do_display(void *index, uchar *pattern, ulong nchars) {
ulong numocc, i, j, *length_snippet, p, len;
uchar *snippet_text;
int error;
len = strlen((char *)pattern) + 2*nchars;
error= display (index, pattern, strlen((char *)pattern), nchars, &numocc,
&snippet_text, &length_snippet);
IFERROR (error);
fprintf (stdout, "Pattern: %s # Snippets: %lu\n\n\n", pattern, numocc);
for (i = 0, p = 0; i < numocc; i++, p+=len) {
fprintf (stdout, "Snippet length: %lu\nText: ", length_snippet[i]);
for(j=0; j < length_snippet[i]; j++)
output_char(snippet_text[p+j]);
fprintf (stdout, "\n\n");
}
if (numocc) {
free (length_snippet);
free (snippet_text);
}
error = free_index(index);
IFERROR(error);
exit(0);
}
int main(int argc, char *argv[]) {
char *infile, *outfile;
uchar *text;
char *params = NULL;
ulong text_len;
void *index;
int error, i;
double start, end;
if (argc < 3) print_usage(argv[0]);
if (argc > 3) {
int nchars, len;
nchars = argc-3;
for(i=2;i<argc;i++)
nchars += strlen(argv[i]);
params = (char *) malloc((nchars+1)*sizeof(char));
params[nchars] = '\0';
nchars = 0;
for(i=3;i<argc;i++) {
len = strlen(argv[i]);
strncpy(params+nchars,argv[i],len);
params[nchars+len] = ' ';
nchars += len+1;
}
params[nchars-1] = '\0';
}
infile = argv[1];
outfile = argv[2];
start = getTime();
error = read_file(infile, &text, &text_len);
IFERROR(error);
error = build_index(text, text_len, params, &index);
IFERROR(error);
error = save_index(index, outfile);
IFERROR(error);
end = getTime();
fprintf(stderr, "Building time: %.3f secs\n", end-start );
ulong index_len;
index_size(index, &index_len);
fprintf(stdout,"Input: %lu bytes --> Output %lu bytes.\n", text_len, index_len);
fprintf(stdout,"Overall compression --> %.2f%% (%.2f bits per char).\n\n",
(100.0*index_len)/text_len, (index_len*8.0)/text_len);
error = free_index(index);
IFERROR(error);
exit(0);
}
void do_count(void *index, uchar *pattern) {
ulong numocc;
int error;
error = count (index, pattern, strlen((char *)pattern), &numocc);
IFERROR (error);
fprintf(stdout,"Pattern: %s %lu occs\n", pattern, numocc);
error = free_index(index);
IFERROR(error);
exit(0);
}
strlist *readfile(char *filename) {
int fd, n, i;
char line[MAXLEN];
strlist *data = strlist_new();
strlist *ret = data;
INFO("Opening file %s\n", filename);
fd = open(filename, O_RDONLY);
if (fd < 0) {
ERROR("ERROR: could not open %s\n", filename);
IFERROR(perror(""));
return NULL;
}
i = 0; // TODO make sure this doesn't overflow line
while ((n = read(fd, &line[i], 1)) > 0) {
if (line[i] == '\n') {
line[i+1] = '\0';
data = strlist_add(data, line);
i = 0;
continue;
}
i++;
}
close(fd);
return ret;
}
void do_extract(void *index, ulong position, ulong nchars) {
int error;
ulong i, read;
uchar *text;
error = extract(index, position, position+nchars-1, &text, &read);
IFERROR(error);
fprintf(stdout, "Extracted from %lu to %lu\n\n",position, position+nchars-1);
for(i=0; i< read; i++)
output_char(text[i]);
fprintf(stdout,"\n\n");
free(text);
error = free_index(index);
IFERROR(error);
exit(0);
}
void do_locate(void *index, uchar *pattern) {
ulong *occs, numocc, i;
int error;
error = locate (index, pattern, strlen((char *)pattern), &occs, &numocc);
IFERROR (error);
fprintf(stdout,"Pattern: %s occs %lu\n\n", pattern, numocc);
for (i = 0; i < numocc; i++)
fprintf (stdout, "Position: %lu\n", occs[i]);
if (numocc) free (occs);
error = free_index(index);
IFERROR(error);
exit(0);
}
void do_unbuild(void *index, char * extfilename) {
FILE *decfile;
uchar *text;
ulong length, text_len = 0;
int error;
decfile = fopen(extfilename, "wb");
if (decfile == NULL)
{
fprintf (stderr, "Error: cannot open file %s\n", extfilename);
perror ("fm-search");
exit (1);
}
error = get_length(index, &text_len);
IFERROR(error);
error = extract(index, 0, text_len-1, &text, &length);
IFERROR(error);
fprintf(stdout,"Original text size = %lu Kbytes\n", text_len/1024);
error = fwrite(text, sizeof(uchar), length, decfile);
if ((error < 0) || (error < (int) length)) {
fprintf (stderr, "Error: cannot write file %s\n", extfilename);
perror ("fm-search");
exit (1);
}
error = free_index(index);
IFERROR(error);
fclose(decfile);
free(text);
exit(0);
}
void response_set_status_code(response *resp, int code) {
if (!resp)
return;
resp->status_code = code;
if (resp->status_message)
free(resp->status_message);
// TODO set status_message properly here
char *msg;
if (code == 200) {
msg = "OK";
} else if (code == 404) {
msg = "Not Found";
} else {
char err_msg[100] = "";
IFERROR(sprintf(err_msg, "Status code %d not implemented!\n", code));
return;
}
resp->status_message = malloc(sizeof(char)*strlen(msg));
strcpy(resp->status_message, msg);
}
/* Fm-Index Search MAIN */
int main(int argc, char *argv[]) {
char * program_name = argv[0];
char * filename, * extfilename = NULL;
void *index;
uchar * pattern;
ulong nchars, position = 0;
int unbuild, count, locate, extract, display, error;
nchars = 10;
count = locate = extract = unbuild = display = 0;
int next_option;
const char* short_options = "hlce:s:d:n:";
const struct option long_options[] = {
{"help" , 0, NULL, 'h'},
{"locate", 0, NULL, 'l'},
{"count", 0, NULL, 'c'},
{"extract", 1, NULL, 'e'},
{"display", 1, NULL, 's'},
{"unbuild", 1, NULL, 'd'},
{"numchars", 1, NULL, 'n'},
{NULL, 0, NULL, 0}
};
if(argc<3) print_usage(program_name);
do {
next_option = getopt_long(argc, argv, short_options,
long_options, NULL);
switch (next_option) {
case 'l': /* report position */
locate = 1;
break;
case 's': /* display len chars sourronding each occ */
display = 1;
nchars = (ulong) atol(optarg);
break;
case 'd': /* unbuild */
unbuild = 1;
extfilename = (char *) optarg;
break;
case 'e': /* extract */
extract = 1;
position = (ulong) atol(optarg);
break;
case 'c': /* count */
count = 1;
break;
case 'n': /* numchars for extract function */
extract = 1;
nchars = (ulong) atol(optarg);
break;
case '?': /* The user specified an invalid option. */
fprintf(stderr,"Unknown option.\n");
print_usage(program_name);
break;
case -1: /* Done with options. */
break;
default:
print_usage(program_name);
}
}
while (next_option != -1);
if (optind == argc) {
fprintf(stderr,"You must supply a pattern and a .fmi filename\n\n");
print_usage(program_name);
exit(1);
}
/* priorita' extract display locate count */
if(!(extract || unbuild)) /* pattern */
pattern = (uchar *) argv[optind++];
else pattern = NULL;
if (optind == argc) {
fprintf(stderr,"You must supply a pattern and a .fmi filename\n\n");
print_usage(program_name);
exit(1);
}
filename = argv[optind];
error = load_index (filename, &index);
IFERROR (error);
if (unbuild==1)
do_unbuild(index, extfilename);
if (extract==1)
do_extract(index, position, nchars);
if (display==1)
do_display(index, pattern, nchars);
if (locate==1)
do_locate(index, pattern);
do_count(index, pattern);
exit(0);
}
int main(/*int argc, char *argv[]*/) {
INFO("Started\n");
int server_socket, client_socket, port, res, client_len, n;
struct sockaddr_in srv_addr, client_addr;
int iSetOption = 1;
port = SRVPORT;
server_socket = socket(AF_INET, SOCK_STREAM, 0);
setsockopt(server_socket, SOL_SOCKET, SO_REUSEADDR, (char*)&iSetOption,
sizeof(iSetOption));
if (server_socket < 0) {
ERROR("Error opening socket\n");
IFERROR(perror(""));
}
INFO("Made server socket\n");
memset((char *) &srv_addr, 0, sizeof(srv_addr));
srv_addr.sin_family = AF_INET;
srv_addr.sin_port = htons(port);
srv_addr.sin_addr.s_addr = INADDR_ANY;
while ((res = bind(server_socket,
(struct sockaddr *) &srv_addr,
sizeof(srv_addr))) < 0) {
ERROR("Error binding socket\n");
IFERROR(perror(""));
}
listen(server_socket, 5);
INFO("Listening on port %d\n", port);
client_len = sizeof(client_addr);
if (chdir("content")) {
ERROR("Could not chdir to the content directory\n");
IFERROR(perror(""));
exit(1);
}
do {
client_socket = accept(server_socket,
(struct sockaddr *) &client_addr,
(socklen_t *) &client_len);
INFO("Accepted client socket\n");
if (client_socket < 0) {
ERROR("Error on client accept\n");
IFERROR(perror(""));
}
request *req = request_new();
char line[MAXLEN];
while ((n = fd_readline(client_socket, line, MAXLEN)) > 0) {
if (n == MAXLEN-1 && line[MAXLEN-1] != '\n') {
WARN("Line length exceeded MAXLEN %d\n", MAXLEN);
char c;
int dropped = 0;
while ((n = read(client_socket, &c, 1)) > 0) {
dropped += 1;
if (c == '\n')
break;
}
WARN("Skipped to the next line, dropped %d characters\n", dropped);
}
if (is_crlf(line)) { // We've reached the end of the headers
int left_to_read = request_get_content_length(req);
int total_read = 0, len;
if (left_to_read > 0) {
DEBUG("There's %d chars of body to read\n", left_to_read);
// There's a Content-Length so there must be a body
req->body = malloc(sizeof(char) * (left_to_read+1));
len = MAXLEN < left_to_read ? MAXLEN : left_to_read;
len += 1;
while (left_to_read && (n = fd_readline(client_socket, line, len)) > 0) {
DEBUG("Read %d characters of body: %s\n", n, line);
strcpy(&req->body[total_read], line);
total_read += n;
left_to_read -= n;
DEBUG("total_read: %d, left_to_read: %d\n", total_read, left_to_read);
len = MAXLEN < left_to_read ? MAXLEN : left_to_read;
}
req->body[total_read+1] = '\0';
DEBUG("Done reading body in %d chars:\n", total_read);
DEBUG("%s\n", req->body);
respond(&req, client_socket);
continue;
} else {
DEBUG("No body to read, generating response\n");
respond(&req, client_socket);
continue;
}
}
if (request_parse(req, line)) {
WARN("An error occurred while parsing the previous line\n");
}
}
DEBUG("Done with this request! n = %d\n", n);
} while (1);
close(server_socket);
close(client_socket);
return 0;
}
int main(int argc, char *argv[]) {
char *infile, *outfile;
uchar *text;
ulong text_len;
void *index;
int error;
if (argc < 3) print_usage(argv[0]);
infile = argv[1]; // input file
outfile = argv[2]; // output file
error = read_file(infile, &text, &text_len);
IFERROR(error);
/* Possible options:
"-a x": indicates the behaviour of FM-index with the pointer 'text'.
x == 0 FM-index uses 'text' directly to build the suffix array.
This means that you are responsable to allocate 'length+overshoot'
bytes for the text instead of 'length' bytes. You must include
ds_ssort.h. See function read_file();
x == 1 FM-index frees the allocated memory for the 'text'. overshoot
and ds_ssort.h are not necessary.
x == 2 FM-index makes its internal copy of 'text'. After the call,
'text' is available. overshoot and ds_ssort.h are not
necessary.
-B Bsize: where Bsize is the size in Kbytes of level 1 buckets.
-b bsize: where bsize is the size in bytes of level 2 buckets.
bsize must divide Bsize*1024;
-f frequency: where frequency is a number from 0 to 1 that indicates the
frequency of the marked characters.
default "-b 512 -B 16 -f 0.02 -a 1"
Example of some call to build_index():
- build_index(text, text_len, NULL, &index);
uses the default parameters.
- build_index(text, text_len, "-a 1 -f 0.1", &index);
tries to mark 10% of the positions instead of 2% but I cannot reuse 'text'
after this call.
*/
fprintf(stdout, "Building\n");
error = build_index(text, text_len, "-a 1", &index);
IFERROR(error);
ulong index_len;
index_size(index, &index_len);
fprintf(stdout,"Input: %lu bytes --> Output %lu bytes.\n", text_len, index_len);
fprintf(stdout,"Overall compression --> %.2f%% (%.2f bits per char).\n\n",
(100.0*index_len)/text_len, (index_len*8.0)/text_len);
uchar *snippet;
ulong i, readen, from = 11, to = 100, numocc, *occ;
error = extract(index, from, to, &snippet, &readen);
IFERROR(error);
fprintf(stdout, "try extract\n\n");
for(i=0;i<readen;i++)
printf("%c", snippet[i]);
printf("\n");
uchar *pattern = snippet;
fprintf(stdout, "try count\n\n");
error = count (index, pattern, 5, &numocc);
printf("pattern: ");
fwrite(pattern, sizeof(uchar), 5, stdout);
printf(" # occs %lu\n\n",numocc);
fprintf(stdout, "try locate\n\n");
error = locate (index, pattern, 5, &occ, &numocc);
IFERROR (error);
for(i=0;i<numocc;i++)
printf("pos %lu\n", occ[i]);
printf("\n");
free(snippet);
if(numocc) free(occ);
error = save_index(index, outfile);
IFERROR(error);
error = free_index(index);
IFERROR(error);
exit(0);
}
