void bw_to_psi(i64 *n0, i64 *last0, suf_t **psi0, char *fbw, char *flst)
{
FILE *in;
i64 i,j,n,last;
suf_t *psi;
i64 C[SIGMA];
i64 w,c,k;
in = fopen(flst,"r");
if (in == NULL) {
perror("bw_to_psi:");
exit(1);
}
fscanf(in,"%ld",&last);
fclose(in);
for (c=0; c<SIGMA; c++) {
C[c] = 0;
}
in = fopen(fbw,"r");
if (in == NULL) {
perror("bw_to_psi:");
exit(1);
}
n = 0;
while (1) {
display_progressbar("reading ",n,(long int)0);
c = fgetc(in);
if (c == EOF) break;
C[c]++;
n++;
}
rewind(in);
printf("n = %ld last = %ld\n",n,last);
psi = (suf_t *)mymalloc((n+1)*sizeof(suf_t));
for (j=1,c=0; c<SIGMA; c++) {
k = C[c];
C[c] = j;
j += k;
}
for (i = 0; i<=n; i++) {
display_progressbar("computing psi ",i,n);
if (i == last) {
psi[0] = i;
} else {
c = fgetc(in);
// printf("psi[%ld] = %ld\n",C[c],i);
psi[C[c]++] = i;
}
}
fclose(in);
*n0 = n;
*last0 = last;
*psi0 = psi;
}
void bw_to_psi(FILE *out, CSA *csa, char *fbw, char *flst, int *k)
{
FILE *in;
i64 last,i,j;
i64 C2[SIGMA];
i64 c;
in = fopen(flst,"r");
if (in == NULL) {
perror("bw_to_psi:"); exit(1);
}
fscanf(in,"%ld",&last);
fclose(in);
for (c=0; c<SIGMA; c++) {
csa->C[c] = 0;
}
in = fopen(fbw,"r");
if (in == NULL) {
perror("bw_to_psi:"); exit(1);
}
csa->n = 0;
while (1) {
display_progressbar("reading ",csa->n,0L);
c = fgetc(in);
if (c == EOF) break;
csa->C[c]++;
csa->n++;
}
rewind(in);
printf("n = %ld last = %ld\n",csa->n,last);
*k = (blog(csa->n+1)+1+8-1)/8;
for (c=0; c<SIGMA; c++) {
psi[c] = open_diskbuf(out,*k);
}
for (j=1,c=0; c<SIGMA; c++) {
C2[c] = j;
j += csa->C[c];
}
for (i = 0; i<=csa->n; i++) {
display_progressbar("computing psi ",i,csa->n);
if (i == last) {
setint_diskbuf(psi[0], 0, i);
} else {
c = fgetc(in);
setint_diskbuf(psi[c], C2[c]++, i);
}
}
fclose(in);
for (c=0; c<SIGMA; c++) {
close_diskbuf(psi[c]);
}
}
void psi_to_sa(i64 n, i64 last, suf_t *psi, suf_t *sa)
{
i64 i,j,v;
j = last;
for (i=0; i<=n; i++) {
display_progressbar("making sa ",i,n);
v = psi[j];
sa[j] = i;
j = v;
}
sa[0] = n;
}
i64 write_isa(CSA *csa, FILE *f2, int isize)
{
i64 i,n,D2,k;
n = csa->n; D2 = csa->D2; k = csa->k;
writeint(1,k,f2); /* #bytes of integer */
writeint(k,D2,f2); /* interval between two SA values stored explicitly */
isize += 1+k;
for (i = 0; i <= (n-1)/D2; i++) {
display_progressbar("writing isa ",i,(n-1)/D2);
writeint(k,getuint(csa->ISA,i,k),f2);
isize += k;
}
return isize;
}
i64 write_sa(CSA *csa, FILE *f2, int isize)
{
i64 i,n,D,k;
n = csa->n; D = csa->D; k = csa->k;
printf("write_sa n=%ld D=%ld k=%ld\n",n,D,k);
writeint(1,k,f2); /* #bytes of integer */
writeint(k,D,f2); /* interval between two SA values stored explicitly */
isize += 1+k;
for (i = 0; i <= n/D; i++) {
display_progressbar("writing sa ",i,n/D);
// printf("sa[%ld] = %ld\n",i,getuint(sa->SA,i,k));
writeint(k,getuint(csa->SA,i,k),f2);
isize += k;
}
return isize;
}
int l_send_file(lua_State *L) {
int argc = lua_gettop(L);
if (argc!=2)
die("Incorrect number of args to set_baud\n");
char* filename = lua_tostring(L,1);
int chunksize = lua_tonumber(L,2);
FILE* fd = fopen(filename, "r");
struct stat inf;
if (0 != stat(filename,&inf)) {
perror("stat failed: ");
die("");
}
if (fd<0){
perror("couldn't open file:");
die("");
}
unsigned int sz = (unsigned int)inf.st_size;
if (minf->iromsz < sz) {
printf("WARNING: File too big to fit in flash, truncating!\n");
sz = minf->iromsz+1;
}
printf("Downloading %s (%d bytes)\n", filename, sz);
unsigned int maxsz = sz;
//unsigned short sz = PACKED_SIZE(chunksize+7);
/* We add a few bytes.
type of packet? 0x00
0x0 0x0
2 bytes, offset to write at
2 bytes, size to write
*/
char* tmp = calloc(1, chunksize+7);
int len;
unsigned short offset=0;
/* Since chunksize is fixed */
tmp[5]=HIGH_BYTE((unsigned short) chunksize);
tmp[6]=LOW_BYTE((unsigned short) chunksize);
struct packet* response;
struct write_response *rsp;
unsigned short crc;
while (sz) {
len = fread(&tmp[7], 1, chunksize, fd);
crc = byte_sum(&tmp[7], chunksize);
crc = crc & 0x00ff; /* We get a shorted, one byte crc */
tmp[3]=HIGH_BYTE(offset);
tmp[4]=LOW_BYTE(offset);
char* packet = pack_payload(tmp,chunksize+7, HOST2MCU);
write(us->fd, packet, PACKED_SIZE(chunksize+7));
do_dump_packet(packet,PACKED_SIZE(chunksize+7));
free(packet);
response = fetch_packet(us->fd);
rsp = response->payload;
if (rsp->errcode !=0 )
{
fprintf(stderr, "Warning, mcu reports error @0x%hx: %hhx\n", offset, rsp->errcode);
//printf();
}else
{
//tmp[0]
}
if (rsp->crc != (unsigned char) crc )
fprintf(stderr, "Warning, crc error @0x%hx: %hhx vs %hhx\n",
offset,
rsp->crc,
(unsigned char) crc);
do_dump_packet(response->payload, response->size);
sz-= len;
offset+=len;
display_progressbar(maxsz,sz);
}
printf("\n");
}
void csa_new_from_bwt(CSA csa, char *fname, char *fidx, int psi_id, int idx_id, bool coded)
{
int k;
i64 i,j,v,m;
FILE *f2;
i64 psize,isize;
i64 n;
psi_id = csa.id;
if (psi_id >= 0) {
printf("create psi: id=%d\n",psi_id);
}
if (idx_id >= 0) {
printf("create idx: id=%d D=%d D2=%d\n",idx_id,csa.D,csa.D2);
}
psize = 0;
if (psi_id >= 0) {
switch (psi_id & 0x3f) {
case ID_DIFF_GAMMA:
case ID_DIFF_GAMMA_RL:
case ID_DIFF_GAMMA_SPARSE:
case ID_DIFF_GAMMA_RL_SPARSE:
psize = psi1_makeindex(&csa, fname);
printf("n %ld\n",csa.n);
printf("Psi %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
case ID_DIFF_GAMMA_RR:
psize = psi12_makeindex(&csa, fname);
printf("n %ld\n",csa.n);
printf("Psi %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
case ID_BWT_DNA:
psize = lf_dna_makeindex(&csa, fname, coded);
printf("n %ld\n",csa.n);
printf("BW %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
case ID_BWT_BIT:
psize = lf_bit_makeindex(&csa, fname);
printf("n %ld\n",csa.n);
printf("BW %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
case ID_BWT_WT:
case ID_BWT_WT_HUF:
case ID_BWT_WT_DENSE:
case ID_BWT_WT_SPARSE4:
case ID_BWT_WT_RR:
psize = lf_wt_makeindex(&csa, fname);
printf("n %ld\n",csa.n);
printf("BW %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
#if 0
case ID_BWT_HUF:
psize = lf_bwt_makeindex(&csa, fname);
printf("n %ld\n",csa.n);
printf("BW %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
#endif
case ID_SPARSE4:
psize = psi2_makeindex(&csa, fname);
printf("n %ld\n",csa.n);
printf("Psi %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
default:
printf("psi_id = %d\n",psi_id);
exit(1);
}
}
csa.k = (blog(csa.n+1)+1+8-1)/8;
for (i=0; i<SIGMA; i++) csa.CtoA[i] = -1;
// csa.K[-1+1] = 0;
csa.K[-1+1] = 1;
for (m=0,v=1,i=0; i<SIGMA; i++) {
if (csa.C[i]>0) {
csa.AtoC[m] = i;
csa.CtoA[i] = m;
csa.K[m+1] = v;
// printf("i=%ld v = %ld C[i] = %ld\n",i,v,csa.C[i]);
v += csa.C[i];
m++;
}
}
csa.K[m+1] = v;
csa.m = m;
if (csa.D >= csa.n) {
printf("D=%d >= n=%ld\n",csa.D,csa.n);
exit(0);
}
if (csa.D2 >= csa.n) {
printf("D2=%d >= n=%ld\n",csa.D2,csa.n);
exit(0);
}
if (idx_id >= 0) {
n = csa.n;
k = csa.k;
//// compute SA and ISA
if (csa.D > 0) csa.SA = (uchar *) mymalloc(((n-1)/csa.D+1+1)*k);
if (csa.D2 > 0) csa.ISA = (uchar *) mymalloc(((n-1)/csa.D2+1+1)*k);
if (csa.D == 0 && csa.D2 == 0) goto brk;
switch (psi_id & 0x3f) {
case ID_DIFF_GAMMA:
case ID_DIFF_GAMMA_RL:
case ID_DIFF_GAMMA_SPARSE:
case ID_DIFF_GAMMA_RL_SPARSE:
case ID_SPARSE4:
case ID_DIFF_GAMMA_RR:
j = 0;
for (i=0; i<=n; i++) {
display_progressbar("making sa ",i,n);
j = csa.psi(&csa,j);
// sa[j] = i;
if (csa.D > 0 && j % csa.D == 0) {
putuint(csa.SA,j / csa.D,i,k);
}
if (csa.D2 > 0 && i % csa.D2 == 0) {
putuint(csa.ISA,i / csa.D2,j,k);
}
}
// putuint(csa.SA,0,n,k);
break;
case ID_BWT_DNA:
case ID_BWT_BIT:
case ID_BWT_WT:
case ID_BWT_WT_HUF:
case ID_BWT_WT_DENSE:
case ID_BWT_WT_SPARSE4:
case ID_BWT_WT_RR:
case ID_BWT_HUF:
j = 0;
for (i=n-1; i>=0; i--) {
display_progressbar("making sa ",i,n);
v = csa.LF(&csa,j);
// printf("LF[%ld] = %ld\n",j,v);
j = v;
if (csa.D > 0 && j % csa.D == 0) putuint(csa.SA, j/csa.D , i, k);
if (csa.D2 > 0 && i % csa.D2 == 0) putuint(csa.ISA, i/csa.D2, j, k);
}
putuint(csa.SA,0,n,k);
break;
default:
break;
}
brk:
//// write idx
f2 = fopen(fidx,"wb"); /* directory */
if (f2 == NULL) {
perror("csa2_new1: ");
exit(1);
}
isize = 0;
writeint(4,VERSION,f2); /* version */
isize += 4;
writeint(1,ID_HEADER,f2); // header ID
isize += 1;
isize = write_header(&csa, f2, isize);
if (csa.D > 0) {
writeint(1,ID_SA,f2);
isize += 1;
isize = write_sa(&csa, f2, isize);
}
if (csa.D2 > 0) {
writeint(1,ID_ISA,f2);
isize += 1;
isize = write_isa(&csa, f2, isize);
}
fclose(f2);
if (csa.D > 0) free(csa.SA);
if (csa.D2 > 0) free(csa.ISA);
printf("Total %ld bytes (%1.3f bpc)\n",(psize+isize),
(double)(psize+isize)*8/csa.n);
}
free(fidx);
}
int
main(int argc, const char * const argv[])
{
int fd, fdout;
size_t read_size, block_size;
unsigned int i, begin_index;
unsigned char *start, *end, *addr;
size_t size;
int page_size;
off_t offset;
const char *file_format;
const char *dir_format;
int c;
read_size = 128 * 1024 * 1024;
block_size = 512;
begin_index = 0;
file_format = "image%05d.jpg";
dir_format = NULL;
while ((c = getopt(argc, (char * const *) argv, "b:d:f:hi:m:qr:s:vV")) != -1) {
switch (c) {
case 'b':
block_size = atol_suffix(optarg);
break;
case 'd':
dir_format = optarg;
break;
case 'f':
file_format = optarg;
break;
case 'i':
begin_index = atoi(optarg);
break;
case 'm':
max_size = atol_suffix(optarg);
break;
case 'q':
quiet = 1;
break;
case 'r':
read_size = atol_suffix(optarg);
break;
case 's':
ignore_size = atol_suffix(optarg) - 1;
break;
case 'v':
verbose = 1;
break;
case 'V':
display_version_and_exit("recoverjpeg");
default:
usage(c == 'h');
}
}
argc -= optind;
argv += optind;
if (argc != 1) {
usage(0);
}
fd = open(argv[0], O_RDONLY);
if (fd < 0) {
fprintf(stderr,
"recoverjpeg: unable to open %s for reading (%s)\n",
argv[argc - 1], strerror(errno));
exit(1);
}
page_size = getpagesize();
if (read_size % page_size || read_size < max_size) {
if (read_size < max_size) {
read_size = max_size;
}
read_size = (read_size + page_size - 1) / page_size * page_size;
if (!quiet) {
fprintf(stderr, "Adjusted read size to %ld bytes\n", (long) read_size);
}
}
start = end = (unsigned char *) malloc(read_size);
if (start == 0) {
fprintf(stderr,
"recoverjpeg: cannot allocate necessary memory (%s)\n",
strerror(errno));
exit(1);
}
for (i = 0, offset = 0, addr = NULL; addr < end;) {
if (progressbar()) {
display_progressbar(offset, i);
}
if (addr == NULL || (size_t) (start + read_size - addr) < max_size) {
off_t base_offset;
long n;
base_offset = offset / page_size * page_size;
lseek(fd, base_offset, SEEK_SET);
n = read(fd, start, read_size);
if (n < 0) {
fprintf(stderr, "recoverjpeg: unable to read data (%s)\n",
strerror(errno));
exit(1);
}
end = start + n;
addr = start + (offset - base_offset);
}
size = jpeg_size(addr);
if (size > ignore_size) {
size_t n;
const char *buffer = file_name(dir_format, file_format, begin_index + i);
i++;
if (verbose) {
printf("%s %ld bytes\n", buffer, (long) size);
}
fdout = open(buffer, O_WRONLY | O_CREAT, 0666);
if (fdout < 0) {
fprintf(stderr, "Unable to open %s for writing\n", buffer);
exit(1);
}
if ((size_t) write(fdout, addr, size) != size) {
fprintf(stderr, "Unable to write %ld bytes to %s\n",
(long) size, buffer);
exit(1);
}
close(fdout);
n = ((size + block_size - 1) / block_size) * block_size;
addr += n;
offset += n;
}
else {
addr += block_size;
offset += block_size;
}
}
if (progressbar()) {
cleanup_progressbar();
}
if (!quiet) {
printf("Restored %d picture%s\n", i, i > 1 ? "s" : "");
}
/* Free allocated memory to keep valgrind happy */
free(start);
exit(0);
}
void csa_new_from_bwt(int argc, char *argv[])
{
i64 i,j,v,m;
FILE *f2;
i64 psize,isize;
i64 n;
int k;
char *fname,*fidx;
char *p;
int psi_id, idx_id;
CSA csa;
int sigma;
csa.sigma = 256; /* default alphabet size */
csa.k2 = 1;
// for (i=0; i<SIGMA+2; i++) csa.C[i] = 0;
// for (i=0; i<SIGMA; i++) csa.C[i] = 0;
fname = NULL; fidx = NULL;
psi_id = idx_id = -1;
for (i=1; i<argc; i++) {
p = argv[i];
if (p[0] == '-') {
p++;
switch (toupper(p[0])) {
case 'I':
// -I[n]:[D]:[D2]
p++;
idx_id = 0;
csa_options(&csa, p);
break;
case 'P':
// -P[n]:[L]
p++;
psi_id = 0;
psi_options(&csa, p);
break;
case 'C':
// -C[s]
p++;
sigma_options(&csa, p);
break;
default:
printf("??? no such option %s\n",argv[i]);
exit(1);
}
} else {
fname = argv[i];
k = strlen(fname);
fidx = mymalloc(k+5);
sprintf(fidx,"%s.idx",fname);
}
}
if (fname == NULL) {
printf("no input file.\n");
exit(0);
}
printf("sigma = %d k2 = %d\n", csa.sigma, csa.k2);
sigma = csa.sigma;
csa.C = mymalloc(sizeof(*csa.C)*sigma); //
csa.CtoA = mymalloc(sizeof(*csa.CtoA)*sigma); //
csa.AtoC = mymalloc(sizeof(*csa.AtoC)*sigma); //
csa.K = mymalloc(sizeof(*csa.K)*(sigma+2)); //
for (i=0; i<sigma; i++) csa.C[i] = 0;
psi_id = csa.id;
if (psi_id >= 0) {
printf("create psi: id=%d\n",psi_id);
}
if (idx_id >= 0) {
printf("create idx: id=%d D=%d D2=%d\n",idx_id,csa.D,csa.D2);
}
psize = 0;
if (psi_id >= 0) {
switch (psi_id & 0x3f) {
case ID_DIFF_GAMMA:
case ID_DIFF_GAMMA_RL:
case ID_DIFF_GAMMA_SPARSE:
case ID_DIFF_GAMMA_RL_SPARSE:
psize = psi1_makeindex(&csa, fname);
printf("n %ld\n",csa.n);
printf("Psi %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
case ID_DIFF_GAMMA_RR:
psize = psi12_makeindex(&csa, fname);
printf("n %ld\n",csa.n);
printf("Psi %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
case ID_BWT_DNA:
psize = lf_dna_makeindex(&csa, fname);
printf("n %ld\n",csa.n);
printf("BW %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
case ID_BWT_DNA2:
psize = lf_dna2_makeindex(&csa, fname);
printf("n %ld\n",csa.n);
printf("BW %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
case ID_BWT_BIT:
psize = lf_bit_makeindex(&csa, fname);
printf("n %ld\n",csa.n);
printf("BW %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
case ID_BWT_WT:
case ID_BWT_WT_HUF:
case ID_BWT_WT_DENSE:
case ID_BWT_WT_SPARSE4:
case ID_BWT_WT_RR:
psize = lf_wt_makeindex(&csa, fname);
printf("n %ld\n",csa.n);
printf("BW %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
#if 0
case ID_BWT_HUF:
psize = lf_bwt_makeindex(&csa, fname);
printf("n %ld\n",csa.n);
printf("BW %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
#endif
case ID_SPARSE4:
psize = psi2_makeindex(&csa, fname);
printf("n %ld\n",csa.n);
printf("Psi %ld bytes (%1.3f bpc)\n",
psize,(double)psize*8/csa.n);
break;
default:
printf("psi_id = %d\n",psi_id);
exit(1);
}
}
csa.k = (blog(csa.n+1)+1+8-1)/8;
for (i=0; i<sigma; i++) csa.CtoA[i] = -1;
csa.K[-1+1] = 1;
for (m=0,v=1,i=0; i<sigma; i++) {
if (csa.C[i]>0) {
csa.AtoC[m] = i;
csa.CtoA[i] = m;
csa.K[m+1] = v;
// printf("i=%ld v = %ld C[i] = %ld\n",i,v,csa.C[i]);
v += csa.C[i];
m++;
}
}
csa.K[m+1] = v;
csa.m = m;
if (csa.D >= csa.n) {
printf("D=%d >= n=%ld\n",csa.D,csa.n);
exit(0);
}
if (csa.D2 >= csa.n) {
printf("D2=%d >= n=%ld\n",csa.D2,csa.n);
exit(0);
}
if (idx_id >= 0) {
n = csa.n;
k = csa.k;
//// compute SA and ISA
if (csa.D > 0) csa.SA = mymalloc(((n-1)/csa.D+1+1)*k);
if (csa.D2 > 0) csa.ISA = mymalloc(((n-1)/csa.D2+1+1)*k);
if (csa.D == 0 && csa.D2 == 0) goto brk;
switch (psi_id & 0x3f) {
case ID_DIFF_GAMMA:
case ID_DIFF_GAMMA_RL:
case ID_DIFF_GAMMA_SPARSE:
case ID_DIFF_GAMMA_RL_SPARSE:
case ID_SPARSE4:
case ID_DIFF_GAMMA_RR:
j = 0;
for (i=0; i<=n; i++) {
display_progressbar("making sa ",i,n);
j = csa.psi(&csa,j);
// sa[j] = i;
if (csa.D > 0 && j % csa.D == 0) {
putuint(csa.SA,j / csa.D,i,k);
}
if (csa.D2 > 0 && i % csa.D2 == 0) {
putuint(csa.ISA,i / csa.D2,j,k);
}
}
// putuint(csa.SA,0,n,k);
break;
case ID_BWT_DNA:
case ID_BWT_DNA2:
case ID_BWT_BIT:
case ID_BWT_WT:
case ID_BWT_WT_HUF:
case ID_BWT_WT_DENSE:
case ID_BWT_WT_SPARSE4:
case ID_BWT_WT_RR:
case ID_BWT_HUF:
j = 0;
for (i=n-1; i>=0; i--) {
display_progressbar("making sa ",i,n);
v = csa.LF(&csa,j);
// printf("LF[%ld] = %ld\n",j,v);
j = v;
if (csa.D > 0 && j % csa.D == 0) putuint(csa.SA, j/csa.D , i, k);
if (csa.D2 > 0 && i % csa.D2 == 0) putuint(csa.ISA, i/csa.D2, j, k);
}
// putuint(csa.SA,0,n,k);
if (csa.D > 0) putuint(csa.SA,0,n,k); // 2011-12-20
break;
default:
break;
}
brk:
//// write idx
f2 = fopen(fidx,"wb"); /* directory */
if (f2 == NULL) {
perror("csa2_new1: ");
exit(1);
}
isize = 0;
writeint(4,VERSION,f2); /* version */
isize += 4;
writeint(1,ID_HEADER,f2); // header ID
isize += 1;
isize = write_header(&csa, f2, isize);
if (csa.D > 0) {
writeint(1,ID_SA,f2);
isize += 1;
isize = write_sa(&csa, f2, isize);
}
if (csa.D2 > 0) {
writeint(1,ID_ISA,f2);
isize += 1;
isize = write_isa(&csa, f2, isize);
}
fclose(f2);
if (csa.D > 0) free(csa.SA);
if (csa.D2 > 0) free(csa.ISA);
printf("Total %ld bytes (%1.3f bpc)\n",(psize+isize),
(double)(psize+isize)*8/csa.n);
}
free(fidx);
}
void bw_to_psi(FILE *out, CSA *csa, char *fbw, char *flst, int *k)
{
FILE *in;
i64 last,i,j;
i64 *C2;
i64 c;
diskbuf **psi;
int sigma;
int k2;
in = fopen(flst,"r");
if (in == NULL) {
perror("bw_to_psi:"); exit(1);
}
fscanf(in,"%ld",&last);
fclose(in);
sigma = csa->sigma;
k2 = csa->k2;
csa->C = mymalloc(sizeof(*csa->C)*sigma);
C2 = mymalloc(sizeof(*C2)*sigma);
for (c=0; c<sigma; c++) {
csa->C[c] = 0;
}
in = fopen(fbw,"r");
if (in == NULL) {
perror("bw_to_psi:"); exit(1);
}
csa->n = 0;
while (1) {
display_progressbar("reading ",csa->n,0L);
// c = fgetc(in);
c = readint(k2,in);
if (c == EOF) break;
if (c >= sigma) {
printf("bw_to_psi: c = %d sigma = %d\n", c, sigma);
exit(1);
}
csa->C[c]++;
csa->n++;
}
rewind(in);
printf("n = %ld last = %ld\n",csa->n,last);
psi = mymalloc(sizeof(diskbuf)*sigma);
*k = (blog(csa->n+1)+1+8-1)/8;
for (c=0; c<sigma; c++) {
psi[c] = open_diskbuf(out,*k);
}
for (j=1,c=0; c<sigma; c++) {
C2[c] = j;
j += csa->C[c];
}
for (i = 0; i<=csa->n; i++) {
display_progressbar("computing psi ",i,csa->n);
if (i == last) {
setint_diskbuf(psi[0], 0, i);
} else {
// c = fgetc(in);
c = readint(k2,in);
setint_diskbuf(psi[c], C2[c]++, i);
}
}
fclose(in);
for (c=0; c<sigma; c++) {
close_diskbuf(psi[c]);
}
free(psi);
free(C2);
}
i64 psi2_makeindex(CSA *csa, char *fname)
{
i64 psize;
i64 b, b2;
i64 i,j,x;
i64 y,d,w;
int k;
FILE *f1,*f2;
char *fpsi;
//psi1_iterator *pi;
i64 n,m;
i64 nn, mm;
psi2 *ps;
int id;
FILE *out;
diskbuf *psi;
char *fbw, *flst;
sparsearray4 sa;
ps = (psi2 *)csa->psi_struc;
id = csa->id;
k = strlen(fname);
fbw = (char *) mymalloc(k+5);
flst = (char *) mymalloc(k+5);
fpsi = (char *) mymalloc(k+5);
sprintf(fbw,"%s.bw",fname);
sprintf(flst,"%s.lst",fname);
sprintf(fpsi,"%s.psa",fname);
out = create_tmp(0);
bw_to_psi(out,csa,fbw,flst,&k);
m = 0;
for (i=0; i<SIGMA; i++) {
if (csa->C[i] > 0) m++;
}
psi = open_diskbuf(out,k);
ps->last = getint_diskbuf(psi,0);
printf("last = %ld\n",ps->last);
n = csa->n;
mm = n+1; nn = (m+1) * (n+1);
sparsearray4_construct_init(&sa, nn, mm);
y = 0; d = 0;
for (j=0; j<=n; j++) {
display_progressbar("compressing psi ",j,n);
x = getint_diskbuf(psi,j);
if (x <= y) d++;
sparsearray4_construct_set(&sa, j, d*(n+1)+x);
y = x;
}
sparsearray4_construct_end(&sa,0, SDARRAY_SELECT1);
f2 = fopen(fpsi,"wb");
psize = 0;
ps->k = k = (blog(n+1)+1+8-1)/8;
writeint(1,ID_PSI,f2);
writeint(1,k,f2); /* #bytes of integer */
writeint(k,n,f2);
psize += 1+1+k;
writeint(1,id,f2);
psize += 1;
psize += sparsearray4_write(&sa, f2);
printf("size %ld (%1.3f bpc)\n",psize,(double)psize*8 / n);
fclose(f2);
// psi1_iterator_remove(pi);
close_diskbuf(psi);
fclose(out);
remove_tmp(0);
psi2_read(csa, fpsi);
free(fpsi);
free(fbw);
free(flst);
return psize;
}
