static inline unsigned long move_vma(struct vm_area_struct * vma,
unsigned long addr, unsigned long old_len, unsigned long new_len)
{
struct vm_area_struct * new_vma;
new_vma = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
if (new_vma) {
unsigned long new_addr = get_unmapped_area(addr, new_len);
if (new_addr && !move_page_tables(current->mm, new_addr, addr, old_len)) {
*new_vma = *vma;
new_vma->vm_start = new_addr;
new_vma->vm_end = new_addr+new_len;
new_vma->vm_offset = vma->vm_offset + (addr - vma->vm_start);
if (new_vma->vm_file)
new_vma->vm_file->f_count++;
if (new_vma->vm_ops && new_vma->vm_ops->open)
new_vma->vm_ops->open(new_vma);
insert_vm_struct(current->mm, new_vma);
merge_segments(current->mm, new_vma->vm_start, new_vma->vm_end);
do_munmap(addr, old_len);
current->mm->total_vm += new_len >> PAGE_SHIFT;
if (new_vma->vm_flags & VM_LOCKED) {
current->mm->locked_vm += new_len >> PAGE_SHIFT;
make_pages_present(new_vma->vm_start,
new_vma->vm_end);
}
return new_addr;
}
static int mmap_mem(struct inode * inode, struct file * file,
unsigned long addr, size_t len, int prot, unsigned long off)
{
struct vm_area_struct * mpnt;
if (off & 0xfff || off + len < off)
return -ENXIO;
if (x86 > 3 && off >= high_memory)
prot |= PAGE_PCD;
if (remap_page_range(addr, off, len, prot))
return -EAGAIN;
/* try to create a dummy vmm-structure so that the rest of the kernel knows we are here */
mpnt = (struct vm_area_struct * ) kmalloc(sizeof(struct vm_area_struct), GFP_KERNEL);
if (!mpnt)
return 0;
mpnt->vm_task = current;
mpnt->vm_start = addr;
mpnt->vm_end = addr + len;
mpnt->vm_page_prot = prot;
mpnt->vm_share = NULL;
mpnt->vm_inode = inode;
inode->i_count++;
mpnt->vm_offset = off;
mpnt->vm_ops = NULL;
insert_vm_struct(current, mpnt);
merge_segments(current->mmap, NULL, NULL);
return 0;
}
/* set new protection flags on an address range */
void VG_(mprotect_range)(Addr a, UInt len, UInt prot)
{
Segment *s, *next;
static const Bool debug = False || mem_debug;
if (debug)
VG_(printf)("mprotect_range(%p, %d, %x)\n", a, len, prot);
/* Everything must be page-aligned */
vg_assert((a & (VKI_BYTES_PER_PAGE-1)) == 0);
len = PGROUNDUP(len);
VG_(split_segment)(a);
VG_(split_segment)(a+len);
for(s = VG_(SkipList_Find)(&sk_segments, &a);
s != NULL && s->addr < a+len;
s = next)
{
next = VG_(SkipNode_Next)(&sk_segments, s);
if (s->addr < a)
continue;
s->prot = prot;
}
merge_segments(a, len);
}
static int mmap_zero(struct inode * inode, struct file * file,
unsigned long addr, size_t len, int prot, unsigned long off)
{
struct vm_area_struct *mpnt;
if (prot & PAGE_RW)
return -EINVAL;
if (zeromap_page_range(addr, len, prot))
return -EAGAIN;
/*
* try to create a dummy vmm-structure so that the
* rest of the kernel knows we are here
*/
mpnt = (struct vm_area_struct *)kmalloc(sizeof(*mpnt), GFP_KERNEL);
if (!mpnt)
return 0;
mpnt->vm_task = current;
mpnt->vm_start = addr;
mpnt->vm_end = addr + len;
mpnt->vm_page_prot = prot;
mpnt->vm_share = NULL;
mpnt->vm_inode = NULL;
mpnt->vm_offset = off;
mpnt->vm_ops = NULL;
insert_vm_struct(current, mpnt);
merge_segments(current->mmap, ignoff_mergep, inode);
return 0;
}
/*
* ensure page tables exist
* mark page table entries with shm_sgn.
*/
static int shm_map (struct vm_area_struct *shmd)
{
unsigned long tmp;
/* clear old mappings */
do_munmap(shmd->vm_start, shmd->vm_end - shmd->vm_start);
/* add new mapping */
tmp = shmd->vm_end - shmd->vm_start;
if ((current->rlim[RLIMIT_AS].rlim_cur < RLIM_INFINITY) &&
((current->mm->total_vm << PAGE_SHIFT) + tmp
> current->rlim[RLIMIT_AS].rlim_cur))
return -ENOMEM;
current->mm->total_vm += tmp >> PAGE_SHIFT;
insert_vm_struct(current->mm, shmd);
merge_segments(current->mm, shmd->vm_start, shmd->vm_end);
return 0;
}
void VG_(map_file_segment)(Addr addr, UInt len, UInt prot, UInt flags,
UInt dev, UInt ino, ULong off, const Char *filename)
{
Segment *s;
static const Bool debug = False || mem_debug;
Bool recycled;
if (debug)
VG_(printf)("map_file_segment(%p, %d, %x, %x, %4x, %d, %ld, %s)\n",
addr, len, prot, flags, dev, ino, off, filename);
/* Everything must be page-aligned */
vg_assert((addr & (VKI_BYTES_PER_PAGE-1)) == 0);
len = PGROUNDUP(len);
/* First look to see what already exists around here */
s = VG_(SkipList_Find)(&sk_segments, &addr);
if (s != NULL && s->addr == addr && s->len == len) {
/* This probably means we're just updating the flags */
recycled = True;
recycleseg(s);
/* If we had a symtab, but the new mapping is incompatible, then
free up the old symtab in preparation for a new one. */
if (s->symtab != NULL &&
(!(s->flags & SF_FILE) ||
!(flags & SF_FILE) ||
s->dev != dev ||
s->ino != ino ||
s->offset != off)) {
VG_(symtab_decref)(s->symtab, s->addr, s->len);
s->symtab = NULL;
}
} else {
recycled = False;
VG_(unmap_range)(addr, len);
s = VG_(SkipNode_Alloc)(&sk_segments);
s->addr = addr;
s->len = len;
s->symtab = NULL;
}
s->flags = flags;
s->prot = prot;
s->dev = dev;
s->ino = ino;
s->offset = off;
if (filename != NULL)
s->filename = VG_(arena_strdup)(VG_AR_CORE, filename);
else
s->filename = NULL;
if (debug) {
Segment *ts;
for(ts = VG_(SkipNode_First)(&sk_segments);
ts != NULL;
ts = VG_(SkipNode_Next)(&sk_segments, ts))
VG_(printf)("list: %8p->%8p ->%d (0x%x) prot=%x flags=%x\n",
ts, ts->addr, ts->len, ts->len, ts->prot, ts->flags);
VG_(printf)("inserting s=%p addr=%p len=%d\n",
s, s->addr, s->len);
}
if (!recycled)
VG_(SkipList_Insert)(&sk_segments, s);
/* If this mapping is of the beginning of a file, isn't part of
Valgrind, is at least readable and seems to contain an object
file, then try reading symbols from it. */
if ((flags & (SF_MMAP|SF_NOSYMS)) == SF_MMAP &&
s->symtab == NULL) {
if (off == 0 &&
filename != NULL &&
(prot & (VKI_PROT_READ|VKI_PROT_EXEC)) == (VKI_PROT_READ|VKI_PROT_EXEC) &&
len >= VKI_BYTES_PER_PAGE &&
s->symtab == NULL &&
VG_(is_object_file)((void *)addr))
{
s->symtab = VG_(read_seg_symbols)(s);
if (s->symtab != NULL) {
s->flags |= SF_DYNLIB;
}
} else if (flags & SF_MMAP) {
const SegInfo *info;
/* Otherwise see if an existing symtab applies to this Segment */
for(info = VG_(next_seginfo)(NULL);
info != NULL;
info = VG_(next_seginfo)(info)) {
if (VG_(seg_overlaps)(s, VG_(seg_start)(info), VG_(seg_size)(info)))
{
s->symtab = (SegInfo *)info;
VG_(symtab_incref)((SegInfo *)info);
}
}
}
}
/* clean up */
merge_segments(addr, len);
}
int
main (int argc, const char *argv[])
{
merge_segments(stdin, stdout);
exit (EXIT_SUCCESS);
}
/*! Read a multipart/form-data submission and write a tsv table of parameters
*/
int
main (void)
{
FILE *fp; /* file pointer from which to read input */
int lines_processed; /* the number of lines processed */
das_config config; /* the DAS configuration */
char list_id[7]; /* user list id */
char unsorted_contents_fn[MAX_LINE_LENGTH]; /* file name for file defining the data set */
char coord_field_fn[MAX_LINE_LENGTH]; /* file name for file with the coords and fields */
char contents_fn[MAX_LINE_LENGTH]; /* file name for file with the field data only */
char rdcontents_fn[MAX_LINE_LENGTH]; /* file name for file with the field data only, duplicates removed */
int fn_length; /* length of most recently constructed file name */
FILE *unsorted_contents; /* the file with the unsorted contents */
FILE *coord_field; /* the file with the unsorted contents */
FILE *contents; /* the file with the contents */
FILE *rdcontents; /* the file with the contents sorted with duplicates removed */
int fclose_return; /* return value of fclose (0 if okay) */
int dataset = 0; /* the data set from which to select data */
char dataset_fn[MAX_LINE_LENGTH]; /* file name for file definig the data set */
FILE *dataset_file; /* the file defining the data set */
char line[FT_TSV_LINE_LENGTH]; /* a buffer of the line input */
char *read_check; /* a pointer to the read string returned by fgets */
int cols_read; /* the columns read from a scanned line */
int ncoords = 0; /* the number of coords encountered */
char url[MAX_URLBASE_LENGTH]; /* url of page giving a fiber overview */
int return_code = 0; /* return code from segment loading */
segment_def *seg_defs = NULL; /* segment definition structure */
int num_segs = 0; /* number of segment defs in the structure */
field_node_pointer f0 = NULL; /* pointer to the list of field nodes */
field_node /*@dependent@*/ *f = NULL; /* pointer to the current field node */
double ra, dec; /* ra and dec provided */
printf ("Content-Type:text/html\n\n");
printf ("<link rel=\"stylesheet\" type=\"text/css\" href=\"../css/sdssdas.css\"/>\n");
printf ("<html>\n");
printf ("<head><title>User Posted Coordinate List</title></head>\n");
printf ("<body>\n");
(void) fflush(stdout);
fp = stdin;
read_config(&config, DAS_CONFIG_FILE);
lines_processed = process_post (fp,
print_line, print_line,
config, list_id, &dataset);
/* Create a TSV with the coordinates and fields */
fn_length = snprintf(unsorted_contents_fn, MAX_LINE_LENGTH,
"%s/userlist-%s/orig_contents.tsv",
config.scratch_root, list_id);
assert(fn_length < MAX_LINE_LENGTH);
fn_length = snprintf(coord_field_fn, MAX_LINE_LENGTH,
"%s/userlist-%s/coord_field.tsv",
config.scratch_root, list_id);
assert(fn_length < MAX_LINE_LENGTH);
fn_length = snprintf(contents_fn, MAX_LINE_LENGTH,
"%s/userlist-%s/unsorted_contents.tsv",
config.scratch_root, list_id);
assert(fn_length < MAX_LINE_LENGTH);
fn_length = snprintf(rdcontents_fn, MAX_LINE_LENGTH,
"%s/userlist-%s/contents.tsv",
config.scratch_root, list_id);
assert(fn_length < MAX_LINE_LENGTH);
/* Create the TSV list containing only data from the specified set */
unsorted_contents = fopen(unsorted_contents_fn, "r");
if (unsorted_contents == NULL)
{
fprintf(stderr,"SDSSDAS could not open %s for reading\n",
unsorted_contents_fn);
exit(EXIT_FAILURE);
}
coord_field = fopen(coord_field_fn, "w");
if (coord_field == NULL)
{
fprintf(stderr,"SDSSDAS could not open %s for writing\n",
coord_field_fn);
exit(EXIT_FAILURE);
}
contents = fopen(contents_fn, "w");
if (coord_field == NULL)
{
fprintf(stderr,"SDSSDAS could not open %s for writing\n",
contents_fn);
exit(EXIT_FAILURE);
}
/* read the reference file */
fn_length = snprintf(dataset_fn, MAX_LINE_LENGTH,
"%s", config.astlimits_fname);
assert(fn_length < MAX_LINE_LENGTH);
return_code = load_segment_defs(dataset_fn, &seg_defs, &num_segs);
/* write the files with the fields */
fprintf(coord_field,"#ra\tdec\trun\trerun\tcamcol\tfield\trow\tcol\n");
fprintf(contents,"#run\trerun\tcamcol\tfield0\tnFields\n");
do
{
read_check = fgets (line, FT_TSV_LINE_LENGTH, unsorted_contents);
ra = 0.0;
dec = 0.0;
cols_read = (read_check == line && (line[0] != '#')) ?
sscanf(line, "%lf%lf", &ra, &dec) : 0;
if (cols_read == 2)
{
(void) matching_fields(ra, dec, seg_defs, num_segs, &f0);
for(f = (field_node *)f0; f != NULL; f = (field_node *)f->next)
{
fprintf(coord_field,"%f\t%f\t%d\t%d\t%d\t%d\t%f\t%f\n",
ra, dec, f->run, f->rerun, f->camcol, f->field,
f->row, f->col);
fprintf(contents,"%d\t%d\t%d\t%d\t1\n",
f->run, f->rerun, f->camcol, f->field);
}
free_field_list(&f0);
}
}
while(read_check == line);
/* Free the structure loaded from the reference file */
free(seg_defs);
fclose_return = fclose(contents);
if (fclose_return != 0)
{
fprintf(stderr,"SDSSDAS error closing %s\n", contents_fn);
exit(EXIT_FAILURE);
}
(void) fflush(contents);
fclose_return = fclose(coord_field);
if (fclose_return != 0)
{
fprintf(stderr,"SDSSDAS error closing %s\n", coord_field_fn);
exit(EXIT_FAILURE);
}
(void) fflush(coord_field);
fclose_return = fclose(unsorted_contents);
if (fclose_return != 0)
{
fprintf(stderr,"SDSSDAS error closing %s\n",
unsorted_contents_fn);
exit(EXIT_FAILURE);
}
/* Sort the resultant fields */
contents = fopen(contents_fn, "r");
if (contents == NULL)
{
fprintf(stderr,"SDSSDAS could not open %s for reading\n", contents_fn);
exit(EXIT_FAILURE);
}
rdcontents = fopen(rdcontents_fn, "w");
if (rdcontents == NULL)
{
fprintf(stderr,"SDSSDAS could not open %s for reading\n", rdcontents_fn);
exit(EXIT_FAILURE);
}
merge_segments(contents, rdcontents);
fclose_return = fclose(rdcontents);
if (fclose_return != 0)
{
fprintf(stderr,"SDSSDAS error closing %s\n", rdcontents_fn);
exit(EXIT_FAILURE);
}
fclose_return = fclose(contents);
if (fclose_return != 0)
{
fprintf(stderr,"SDSSDAS error closing %s\n", contents_fn);
exit(EXIT_FAILURE);
}
printf ("<h1>Fields containing specified coordinates</h1>\n" );
/* Print list summary */
printf ("<h2>List summary</h2>\n");
printf ("<div class=\"list-summary\">\n");
printf ("<table class=\"list-summary\">\n");
printf ("<tr><th>List serial number</th><td>%s</td></tr>\n", list_id);
printf ("<tr><th>Input lines processed</th><td>%d</td></tr>\n", lines_processed);
printf ("</table>\n");
printf ("</div>\n");
/* Download choices */
printf ("<h2>Download selection</h2>\n");
printf ("<p class=\"left\">You can use <a href=\"%s/dl_request_form?list=%s\">", config.cgi_url, list_id);
printf ("this form</a> to generate lists of files that can be used by wget or rsync for\n");
printf ("mass retrieval of data. It allows selection of specific filters, \n");
printf ("types of files for download, and download method.</p>");
/* Print a table of uploaded segments */
printf ("<h2>Contents of the list</h2>\n");
rdcontents = fopen(rdcontents_fn, "r");
if (rdcontents == NULL)
{
fprintf(stderr,"SDSSDAS could not open %s for reading\n", contents_fn);
exit(EXIT_FAILURE);
}
segmentlist_table(rdcontents, stdout, list_id, config);
fclose_return = fclose(rdcontents);
if (fclose_return != 0)
{
fprintf(stderr,"SDSSDAS error closing %s\n", rdcontents_fn);
exit(EXIT_FAILURE);
}
printf ("</body>\n");
printf ("</html>\n");
exit(EXIT_SUCCESS);
}
unsigned long do_mmap(struct file *file,unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long off)
{
int error;
struct vm_area_struct *vma;
if(len <= 0)
return -EINVAL;
if((len = PAGE_ALIGN(len)) == 0)
return addr;
if(addr > PAGE_OFFSET || len > PAGE_OFFSET || (addr + len) > PAGE_OFFSET)
return -EINVAL;
if(!file)
{
switch (flags & MAP_TYPE)
{
case MAP_SHARED:
if((prot & PROT_WRITE) && (file->f_mode & FILE_WRITE))
return -EACCES;
break;
case MAP_PRIVATE:
if(!(file->f_mode & FILE_READ))
return -EACCES;
default:
return -EINVAL;
}
if(file->f_inode->i_count > 0 && flags & MAP_DENYWRITE)
return -ETXTBSY;
}
else if((flags & MAP_TYPE) != MAP_PRIVATE)
return -EINVAL;
if(flags & MAP_FIXED)
{
if(addr & ~ PAGE_MASK)
return -EINVAL;
if(len > PAGE_OFFSET || addr + len > PAGE_OFFSET)
return -EINVAL;
}
else
{
addr = get_unmmapped_area(len);
if(!addr)
return -ENOMEM;
}
if(file && (!file->f_op || !file->f_op->mmap))
return -ENODEV;
vma = (struct vm_area_struct *)kmalloc(sizeof(struct vm_area_struct),GFP_KERNEL);
vma->vm_task = current;
vma->vm_start = addr;
vma->vm_end = addr + len;
vma->vm_flags = prot & (VM_READ | VM_WRITE | VM_EXEC);
vma->vm_flags |= flags & (VM_GROWSDOWN | VM_DENYWRITE | VM_EXECUTABLE);
if(file)
{
if(file->f_mode & FILE_READ)
vma->vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
if(flags & MAP_SHARED)
vma->vm_flags |= VM_SHARED | VM_MAYSHARE;
if(file->f_mode & FILE_WRITE)
vma->vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
}
else
vma->vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
// vma->vm_page_prot = protection_map[vma->vm_flags & 0X0F];
vma->vm_ops = NULL;
vma->vm_offset = off;
vma->vm_inode = NULL;
vma->vm_pte = 0;
do_munmap(addr, len);
if(file)
error = file->f_op->mmap(file->f_inode, file, vma);
else
error = anon_map(NULL, NULL, vma);
if(error)
{
kfree(vma);
return error;
}
insert_vm_struct(current, vma);
merge_segments(current, vma->vm_start, vma->vm_end);
return addr;
return 0;
}
int main(int argc, char *argv[]) {
interpret_args(argc, argv);
// Open files and set info
open_source_file(filename);
dest_info.samplerate = 44100;
dest_info.channels = 2;
dest_info.format = SF_FORMAT_WAV ^ SF_FORMAT_PCM_16 ^ SF_ENDIAN_FILE;
open_dest_file(dest_file, &dest_info);
if (verbose) {
logger(NOTICE, "Sample rate: %d", source_info.samplerate);
logger(NOTICE, "Frames: %d", (int) source_info.frames);
logger(NOTICE, "Channels: %d", source_info.channels);
logger(NOTICE, "%s minutes long.", prettify_seconds(source_info.frames/(double) source_info.samplerate, 0));
}
// Calculate Root-Mean-Square of source sound
FRAMES_IN_RMS_FRAME = (source_info.samplerate*RMS_FRAME_DURATION)/1000; // (44100/1000)*20
RMS_FRAME_COUNT = ceil(source_info.frames/(float) FRAMES_IN_RMS_FRAME);
float *rms = malloc(2*RMS_FRAME_COUNT*sizeof(float));
calculate_rms(rms);
if (verbose) {
logger(NOTICE, "Calculated RMS!");
}
// Calculate RMS-derived features of long (1 second) frames
FRAMES_IN_LONG_FRAME = (source_info.samplerate*LONG_FRAME_DURATION)/1000; // (44100/1000)*20
LONG_FRAME_COUNT = ceil(source_info.frames/(float) FRAMES_IN_LONG_FRAME);
RMS_FRAMES_IN_LONG_FRAME = LONG_FRAME_DURATION/RMS_FRAME_DURATION;
float *mean_rms = malloc(2*LONG_FRAME_COUNT*sizeof(float));
float *variance_rms = malloc(2*LONG_FRAME_COUNT*sizeof(float));
float *norm_variance_rms = malloc(2*LONG_FRAME_COUNT*sizeof(float));
float *mler = malloc(2*LONG_FRAME_COUNT*sizeof(float));
calculate_features(rms, mean_rms, variance_rms, norm_variance_rms, mler);
logger(NOTICE, "Calculated features!");
// CLASSIFY
bool *is_music = malloc(2*LONG_FRAME_COUNT*sizeof(bool));
// Decide whether a given second segment is music or speech,
// based on the MLER value and the Upper Music Threshold
classify_segments(is_music, mler);
// The music-ness of the frame equals the average music-ness of itself and
// the two previous and next frames
average_musicness(is_music);
// Merge smaller segments with larger segments
segment *merged_segments = malloc(LONG_FRAME_COUNT*sizeof(segment));
int merged_segment_count = merge_segments(is_music, merged_segments);
// Finally, we write only the speech sections to the destination file
write_speech_to_file(merged_segments, merged_segment_count);
// Close files and free memory
bool file_close_success = finalize_files();
logger(WARNING, "Successfully generated %s!", output_path);
free(mean_rms);
free(variance_rms);
free(norm_variance_rms);
free(mler);
free(merged_segments);
free(is_music);
free(rms);
return !file_close_success;
}
