static int afm_create(AFFILE *af)
{
if (af_update_seg (af, AF_RAW_IMAGE_FILE_EXTENSION, 0, (const u_char *)SPLITRAW_DEFAULT_EXTENSION,
strlen(SPLITRAW_DEFAULT_EXTENSION))) {
(*af->error_reporter)("split_raw_read_write_setup: %s: failed to write %s\n",
af->fname, AF_RAW_IMAGE_FILE_EXTENSION);
afm_close(af); // close the sub-file
return -1; // we failed
}
af_set_pagesize(af,AFM_DEFAULT_PAGESIZE);
af_update_seg(af,AF_AFF_FILE_TYPE,0,(const u_char *)"AFM",3);
return 0;
}
void large_file_test()
{
int pagesize = 1024*1024; // megabyte sized segments
int64_t num_segments = 5000;
int64_t i;
char fn[1024];
printf("Large file test... Creating a %"I64d"MB file...\n",pagesize*num_segments/(1024*1024));
filename(fn,sizeof(fn),"large_file");
AFFILE *af = af_open(fn,O_CREAT|O_RDWR|O_TRUNC,0666);
unsigned char *buf = (unsigned char *)malloc(pagesize);
memset(buf,'E', pagesize);
af_enable_compression(af,opt_compression_type,opt_compression_level);
af_set_pagesize(af,pagesize);
af_set_maxsize(af,(int64_t)pagesize * 600);
for(i=0;i<num_segments;i++){
sprintf((char *)buf,"%"I64d" page is put here",i);
if(i%25==0) printf("\rWriting page %"I64d"\r",i);
if(af_write(af,buf,pagesize)!=pagesize){
err(1,"Can't write page %"I64d,i);
}
}
printf("\n\n");
/* Now let's just read some test locations */
for(i=0;i<num_segments;i+=num_segments/25){ // check a few places
int r;
af_seek(af,pagesize*i,SEEK_SET);
r = af_read(af,buf,1024); // just read a bit
if(r!=1024){
err(1,"Tried to read 1024 bytes; got %d\n",r);
}
if(atoi((char *)buf)!=i){
err(1,"at page %"I64d", expected %"I64d", got %s\n",i,i,buf);
}
printf("Page %"I64d" validates\n",i);
}
af_close(af);
if(unlink("large_file.aff")){
err(1,"Can't delete large_file.aff");
}
printf("Large file test passes\n");
}
AFFILE *open_testfile(const char *base,int wipe)
{
int flags = O_CREAT|O_RDWR;
char fn[1024];
filename(fn,sizeof(fn),base);
printf("%s = %s\n",base,fn);
if(wipe){
unlink(fn); // make sure it is gone
flags |= O_TRUNC;
}
AFFILE *af = af_open(fn,flags,0666);
if(!af) err(1,"af_open");
if(wipe){
af_enable_compression(af,opt_compression_type,opt_compression_level);
af_set_pagesize(af,1024);
af_set_maxsize(af,(int64_t)65536); // force splitting of raw and afd files
}
return af;
}
int af_write(AFFILE *af,unsigned char *buf,size_t count)
{
AF_WRLOCK(af);
if (af_trace){
fprintf(af_trace,"af_write(af=%p,buf=%p,count=%zd) pos=%"I64d"\n", af,buf,count,af->pos);
}
/* Invalidate caches */
af_invalidate_vni_cache(af);
/* vnode write bypass:
* If a write function is defined, use it and avoid the page and cache business.
*/
if (af->v->write){
int r = (af->v->write)(af, buf, af->pos, count);
if(r>0){
af->pos += r;
af->bytes_written += r;
}
if(af->pos >= af->image_size) af->image_size = af->pos;
AF_UNLOCK(af);
return r;
}
/* If no pagesize has been set, go with the default pagesize */
if(af->image_pagesize==0){
if(af_set_pagesize(af,AFF_DEFAULT_PAGESIZE)){
AF_UNLOCK(af);
return -1;
}
}
int64_t offset = af->pos; // where to start
/* If the correct segment is not loaded, purge the current segment */
int64_t write_page = offset / af->image_pagesize;
if(af->pb && af->pb->pagenum!=write_page){
af_cache_flush(af);
af->pb = 0;
}
int write_page_offset = (int)(offset % af->image_pagesize);
/* Page Write Bypass:
* If no data has been written into the current page buffer,
* and if the position of the stream is byte-aligned on the page buffer,
* and if an entire page is being written,
* just write it out and update the pointers, then return.
*/
if(af->pb==0 && af->image_pagesize==(unsigned)count && write_page_offset == 0){
// copy into cache if we have this page anywhere in our cache
af_cache_writethrough(af,write_page,buf,count);
int ret = af_update_page(af,write_page,buf,count);
if(ret==0){ // no error
af->pos += count;
if(af->pos > af->image_size) af->image_size = af->pos;
AF_UNLOCK(af);
return count;
}
AF_UNLOCK(af);
return -1; // error
}
/* Can't use high-speed optimization; write through the cache */
int total = 0;
while(count>0){
/* If no page is loaded, or the wrong page is loaded, load the correct page */
int64_t pagenum = offset / af->image_pagesize; // will be the segment we want
if(af->pb==0 || af->pb->pagenum != pagenum){
af->pb = af_cache_alloc(af,pagenum);
af->pb->pagebuf_bytes = af->image_pagesize;
assert(af->pb->pagenum == pagenum);
/* Now try to load the page.
* If we can't load it, then we are creating a new page.
*/
if(af_get_page(af,af->pb->pagenum,af->pb->pagebuf, &af->pb->pagebuf_bytes)){
/* Creating a new page; note that we have no bytes in this page */
af->pb->pagebuf_bytes = 0;
}
}
// where writing to
u_int seg_offset = (u_int)(offset - af->pb->pagenum * af->image_pagesize);
// number we can write into
u_int seg_left = af->image_pagesize - seg_offset;
u_int bytes_to_write = count;
if(bytes_to_write > seg_left) bytes_to_write = seg_left;
assert(bytes_to_write >= 0); //
if(bytes_to_write==0) break; // that's all we could get
/* Copy out the bytes for the user */
memcpy(af->pb->pagebuf+seg_offset,buf,bytes_to_write); // copy into the page cache
af->bytes_memcpy += bytes_to_write;
if(af->pb->pagebuf_bytes < seg_offset+bytes_to_write){
af->pb->pagebuf_bytes = seg_offset+bytes_to_write; // it has been extended.
}
buf += bytes_to_write;
offset += bytes_to_write;
count -= bytes_to_write;
total += bytes_to_write;
af->pos += bytes_to_write;
af->pb->pagebuf_valid = 1;
af->pb->pagebuf_dirty = 1;
/* If we wrote out all of the bytes that were left in the segment,
* then we are at the end of the segment, write it back...
*/
if(seg_left == bytes_to_write){
if(af_cache_flush(af)){
AF_UNLOCK(af);
return -1;
}
}
/* If we have written more than the image size, update the image size */
if((uint64_t)offset > af->image_size) af->image_size = offset;
}
/* We have copied all of the user's requested data, so return */
AF_UNLOCK(af);
return total;
}
/* Add a file to the AFF system.
* if fname==0, create a new one and copy over the relevant metadata...
*/
static int afd_add_file(AFFILE *af,const char *fname_)
{
struct afd_private *ap = AFD_PRIVATE(af);
const char *segs_to_copy[] = {AF_BADFLAG,
AF_CASE_NUM,
AF_IMAGE_GID,
AF_ACQUISITION_ISO_COUNTRY,
AF_ACQUISITION_COMMAND_LINE,
AF_ACQUISITION_DATE,
AF_ACQUISITION_NOTES,
AF_ACQUISITION_DEVICE,
AF_ACQUISITION_TECHNICIAN,
AF_DEVICE_MANUFACTURER,
AF_DEVICE_MODEL,
AF_DEVICE_SN,
AF_DEVICE_FIRMWARE,
AF_DEVICE_SOURCE,
AF_CYLINDERS,
AF_HEADS,
AF_SECTORS_PER_TRACK,
AF_LBA_SIZE,
AF_HPA_PRESENT,
AF_DCO_PRESENT,
AF_LOCATION_IN_COMPUTER,
AF_DEVICE_CAPABILITIES,
0};
char fname[MAXPATHLEN+1];
memset(fname,0,sizeof(fname));
if(fname_){
strlcpy(fname,fname_,sizeof(fname));
}
else {
aff_filename(af,fname,sizeof(fname),ap->num_afs);
}
int new_file = access(fname,F_OK)!=0; // Is this a new file?
AFFILE *af2 = af_open(fname,af->openflags|AF_NO_CRYPTO,af->openmode);
if(af2==0){
(*af->error_reporter)("open(%s,%d,%d) failed: %s\n",
fname,af->openflags,af->openmode,strerror(errno));
return -1; // this is bad
}
ap->num_afs += 1;
ap->afs = (AFFILE **)realloc(ap->afs,sizeof(AFFILE *) * ap->num_afs);
ap->afs[ap->num_afs-1] = af2;
if(new_file){
/* Copy over configuration from AFD vnode*/
af_enable_compression(af2,af->compression_type,af->compression_level);
af_set_pagesize(af2,af->image_pagesize); //
af_set_sectorsize(af2,af->image_sectorsize);
af_update_seg(af,AF_AFF_FILE_TYPE,0,(const u_char *)"AFD",3);
/* If this is the second file, copy over additional metadata from first... */
if(ap->num_afs>1){
AFFILE *af0 = ap->afs[0];
memcpy(af2->badflag,af0->badflag,af->image_sectorsize);
af2->bytes_memcpy += af->image_sectorsize;
for(const char **segname=segs_to_copy;*segname;segname++){
unsigned char data[65536]; // big enough for most metadata
size_t datalen = sizeof(data);
unsigned long arg=0;
if(af_get_seg(af0,*segname,&arg,data,&datalen)==0){
int r = af_update_seg(af2,*segname,arg,data,datalen);
if(r!=0){
(*af->error_reporter)("afd_add_file: could not update %s in %s (r=%d)",
*segname,af_filename(af2),r);
}
}
}
}
}
return 0;
}
int aestest()
{
unsigned char keyblock[32];
/* Make a key; doesn't need to be a good key; make it 256 bits */
for(int i=0;i<32;i++){
keyblock[i] = i;
}
AFFILE *af = af_open("crypto.aff",O_CREAT|O_RDWR|O_TRUNC,0666);
if(!af) err(1,"af_open");
if(af_set_aes_key(af,keyblock,256)) err(1,"af_set_aes_key");
af_set_pagesize(af,65536);
/* Now, let's write some data of various sizes */
u_char test[1024],buf[1024],rbuf[1024];
size_t buflen = sizeof(buf);
make_test_seg(test,0);
for(u_int len=0;len<=strlen((const char *)test);len++){
if(af_update_seg(af,"page0",0,test,len)) err(1,"af_update_seg len=%d",len);
/* Now try to read the segment */
memset(buf,0,sizeof(buf));
buflen = sizeof(buf);
if(af_get_seg(af,"page0",0,(unsigned char *)buf,&buflen)){
err(1,"Could not read encrypted segment with length %d.\n",len);
}
if(buflen!=len){
printf("size of returned segment = %zd ",buflen);
printf("(should be %d) \n",len);
exit(0);
}
if(memcmp(buf,test,len)!=0){
printf("does not match\n");
printf(" wanted: %s\n",test);
printf(" got: %s\n",buf);
exit(0);
}
}
if(af_close(af)) err(1,"af_close");
/* Now re-open the file, do not set the encryption key, and see if we can read it */
int r;
memset(buf,0,sizeof(buf));
af = af_open("crypto.aff",O_RDONLY,0666);
buflen = sizeof(buf);
r = af_get_seg(af,"page0",0,(unsigned char *)buf,&buflen);
if(r!=-1) {
errx(1,"Error; attempt to read segment 'encrypted' succeded. It should have failed.");
}
/* Try to read 'encrypted/aes' */
r = af_get_seg(af,"encrypted/aes",0,(unsigned char *)buf,&buflen);
if(memcmp(buf,test,buflen)==0){
errx(1,"Error: segment encrypted/aes wasn't actually encrypted.");
}
af_close(af);
/* Now set the correct encryption key and see if we can read it */
af = af_open("crypto.aff",O_RDONLY,0666);
if(af_set_aes_key(af,keyblock,256)) err(1,"af_set_aes_key");
buflen = sizeof(buf);
memset(buf,0,sizeof(buf));
r = af_get_seg(af,"page0",0,(unsigned char *)buf,&buflen);
if(buflen != strlen((const char *)test)){
errx(1,"Error: Could not read encrypted segment after re-opening file");
}
if(memcmp(buf,test,buflen)!=0) errx(1,"Error: Re-read of file produces wrong data.");
printf("encrypted data read and decrypted: '%s'\n",buf);
/* Try to read a segment that doesn't eixst */
buflen = 0;
if(af_get_seg(af,"encrypted2",0,0,&buflen)==0){
errx(1,"Error: Attempt to get size of non-existant segment 'encrypted2' got %zd\n",buflen);
}
af_close(af);
/* Now set the wrong encryption key and see if we can read it */
memset(buf,0,sizeof(buf));
af = af_open("crypto.aff",O_RDONLY,0666);
keyblock[3] = 42;
if(af_set_aes_key(af,keyblock,256)) err(1,"af_set_aes_key");
buflen = sizeof(buf);
r = af_get_seg(af,"page0",0,(unsigned char *)buf,&buflen);
if(memcmp(buf,test,buflen)==0) errx(1,"Error: Setting wrong key still produces correct data.");
af_close(af);
printf("Basic crypto checks. Now check passphrase....\n");
/* Write the data with a passphrase and try to read it back */
af = af_open("crypto_pass.aff",O_CREAT|O_RDWR|O_TRUNC,0666);
if(!af) err(1,"af_open 3");
af_set_pagesize(af,65536);
if(af_establish_aes_passphrase(af,"yummy")) err(1,"af_establish_aes_passphrase");
if(af_use_aes_passphrase(af,"yummy")) err(1,"af_use_aes_passphrase");
if(af_update_seg(af,"page0",0,(const u_char *)test,strlen((const char *)test))) err(1,"af_update_seg failed at 3");
if(af_close(af)) err(1,"af_close at 3");
/* Now try to read it back */
memset(rbuf,0,sizeof(rbuf));
size_t rbuflen = sizeof(rbuf);
af = af_open("crypto_pass.aff",O_RDONLY,0666);
if(!af) err(1,"af_open 4");
if(af_get_seg(af,"page0",0,(unsigned char *)buf,&buflen)==0){
errx(1,"af_get_seg should have failed and didn't");
}
if(af_use_aes_passphrase(af,"yummy")) err(1,"af_set_passphrase 2");
rbuflen=sizeof(rbuf);
if(af_get_seg(af,"page0",0,(unsigned char *)rbuf,&rbuflen)){
errx(1,"af_get_seg failed");
}
if(rbuflen!=strlen((const char *)test)) errx(1,"Reading encrypted data returned wrong size");
if(memcmp(rbuf,test,rbuflen)!=0) errx(1,"Error: wrong data");
printf("encrypted data read with passphrase 'yummy': %s\n",rbuf);
af_close(af);
/* Try to change the passphrase */
af = af_open("crypto_pass.aff",O_RDWR,0666);
if(!af) err(1,"af_open 5");
if(af_change_aes_passphrase(af,"yummy","dummy")) err(1,"could not change passphrase");
af_close(af);
/* Try to read with new passphrase */
af = af_open("crypto_pass.aff",O_RDONLY,0666);
if(!af) err(1,"af_open 5");
memset(rbuf,0,sizeof(rbuf));
rbuflen = sizeof(rbuf);
if(af_use_aes_passphrase(af,"dummy")) err(1,"af_set_passphrase 2");
rbuflen=sizeof(rbuf);
if(af_get_seg(af,"page0",0,(unsigned char *)rbuf,&rbuflen)){
errx(1,"af_get_seg failed");
}
if(rbuflen!=strlen((const char *)test)) errx(1,"Reading encrypted with new passphrase data returned wrong size");
if(memcmp(rbuf,test,rbuflen)!=0) errx(1,"Error: wrong data");
printf("encrypted data read with new passphrase 'dummy': %s\n",rbuf);
af_close(af);
exit(0);
/* Now try to read with the wrong passphrase */
af = af_open("crypto.aff",O_RDONLY,0666);
if(af_use_aes_passphrase(af,"yummy2")) err(1,"af_set_passphrase 3");
buflen=sizeof(buf);
memset(buf,0,sizeof(buf));
if(af_get_seg(af,"page0",0,(unsigned char *)buf,&buflen)){
printf("Couldn't get data with wrong passphrase (that's good)\n");
}
printf("data read with wrong passphrase: %s\n",buf);
if(buflen>0 && memcmp(buf,test,buflen)==0){
errx(1,"Error: data fetched with wrong passphrase was not scrambled.");
}
af_close(af);
exit(0);
}
void sparse_test()
{
printf("Sparse test...\n");
char buf[1024];
char fn[1024];
uint64_t mult = (uint64_t)3 * (uint64_t)1000000000; // 3GB in
AFFILE *af = af_open(filename(fn,sizeof(fn),"sparse"),O_CREAT|O_RDWR|O_TRUNC,0666);
af_enable_compression(af,opt_compression_type,opt_compression_level);
af_set_maxsize(af,(int64_t)1024*1024*256);
af_set_pagesize(af,1024*1024*16);
for(u_int i=0;i<10;i++){
uint64_t pos = mult*i;
memset(buf,0,sizeof(buf));
snprintf(buf,sizeof(buf),"This is at location=%"I64u"\n",pos);
af_seek(af,pos,SEEK_SET);
af_write(af,(unsigned char *)buf,sizeof(buf));
}
/* Now verify */
for(u_int i=0;i<10;i++){
uint64_t pos = mult*i;
uint64_t q;
af_seek(af,pos,SEEK_SET);
af_read(af,(unsigned char *)buf,sizeof(buf));
char *cc = strchr(buf,'=');
if(!cc){
printf("Garbage read at location %"I64u"\n.",pos);
exit(1);
}
if(sscanf(cc+1,"%"I64u,&q)!=1){
printf("Could not decode value at location %"I64u"(%s)\n",pos,cc+1);
exit(1);
}
if(pos!=q){
printf("Wrong value at location %"I64u"; read %"I64u" in error.\n", pos,q);
exit(1);
}
}
/* Now seek to somewhere that no data has been written and see if we get 0s. */
memset(buf,'g',sizeof(buf));
af_seek(af,mult/2,SEEK_SET);
ssize_t r = af_read(af,(unsigned char *)buf,sizeof(buf));
if(r!=sizeof(buf)){
err(1,"Tried to read %zd bytes at mult/2; got %zd bytes\n",sizeof(buf),r);
}
for(u_int i=0;i<sizeof(buf);i++){
if(buf[i]!=0) err(1,"data error; buf[%d]=%d\n",i,buf[i]);
}
/* Now try to read the last page in the file */
unsigned char big_buf[65536];
af_seek(af,9*mult,SEEK_SET);
r = af_read(af,big_buf,sizeof(big_buf));
if(r!=sizeof(buf)){
errx(1,"Tried to read %zd bytes at the end of the file; got %zd bytes (should get %zd)",
sizeof(big_buf),r,sizeof(buf));
}
/* Now see if we can read past the end of the file */
af_seek(af,11*mult,SEEK_SET);
r = af_read(af,(unsigned char *)buf,sizeof(buf));
if(r!=0) errx(1,"Tried to read past end of file; got %zd bytes (should get 0)",r);
af_close(af);
printf("\nSprase test passes.\n");
printf("=====================\n\n");
}
