void write_mmap(void* mapped_mem, const unsigned long graph_type) { unsigned long* ul_mapped_mem = reinterpret_cast<unsigned long*>(mapped_mem); // Write the mmap type, mmap size, graph size, and graph order to the // mapped memory. ul_mapped_mem[0] = graph_type; ul_mapped_mem[1] = get_mmap_size(); ul_mapped_mem[2] = n_vertices; ul_mapped_mem[3] = n_edges; // Write the base pointer to the memory. uintptr_t* base_ptr = reinterpret_cast<uintptr_t*>(ul_mapped_mem + 4); *base_ptr = reinterpret_cast<uintptr_t>(ul_mapped_mem); // Write the number of edge types to the mapped memory. size_type* n_etypes_ptr = reinterpret_cast<size_type*>(base_ptr + 1); *n_etypes_ptr = n_etypes; // Copy the vertices array to the mapped memory. vertex_t* vertices_ptr = reinterpret_cast<vertex_t*>(n_etypes_ptr + 1); memcpy(vertices_ptr, vertices, n_vertices * sizeof(vertex_t)); // Set up the edge_types arrays in the mapped memory. etype_entry_t* edge_types_ptr = reinterpret_cast<etype_entry_t*>(vertices_ptr + n_vertices); edge_block_t** edge_types_blocks_ptr = reinterpret_cast<edge_block_t**>(edge_types_ptr + n_etypes); // Get the total number of edge blocks. size_type total_edge_blocks = 0; for (size_type i = 0; i < n_etypes; ++i) { total_edge_blocks += edge_types[i].num_blocks; } // Copy the edge blocks to the mmapped memory. edge_block_t* edge_blocks_ptr = reinterpret_cast<edge_block_t*>( edge_types_blocks_ptr + total_edge_blocks); memcpy(edge_blocks_ptr, edge_blocks, total_edge_blocks * sizeof(edge_block_t)); for (size_type i = 0; i < n_etypes; ++i) { edge_types_ptr[i].num_blocks = edge_types[i].num_blocks; edge_types_ptr[i].edge_blocks = edge_types_blocks_ptr; edge_types_blocks_ptr += edge_types[i].num_blocks; } // Get the offset for all the edge block pointers. ptrdiff_t ptr_offset = reinterpret_cast<char*>(edge_blocks_ptr) - reinterpret_cast<char*>(edge_blocks); // Add the edge block pointer offset to all the vertex adjacency list heads. #pragma mta assert parallel for (size_type i = 0; i < n_vertices; ++i) { // Only add the offset if it's a valid pointer. Null values stay null. if (vertices_ptr[i].edge_blocks) { vertices_ptr[i].edge_blocks = reinterpret_cast<edge_block_t*>( reinterpret_cast<char*>(vertices[i].edge_blocks) + ptr_offset); } else { vertices_ptr[i].edge_blocks = 0; } } // Add the edge block pointer offset to all the pointers in the edge // block type arrays. #pragma mta assert parallel #pragma mta assert nodep for (size_type i = 0; i < n_etypes; ++i) { #pragma mta assert parallel #pragma mta assert nodep for (size_type j = 0; j < edge_types_ptr[i].num_blocks; ++j) { edge_types_ptr[i].edge_blocks[j] = reinterpret_cast<edge_block_t*>( reinterpret_cast<char*>(edge_types[i].edge_blocks[j]) + ptr_offset); } } // Add the edge block pointer offset to all the vertex adjacency list // heads. #pragma mta assert parallel for (size_type i = 0; i < total_edge_blocks; ++i) { // Only add the offset if it's a valid pointer. Null values stay null. if (edge_blocks[i].next) { edge_blocks_ptr[i].next = reinterpret_cast<edge_block_t*>( reinterpret_cast<char*>(edge_blocks[i].next) + ptr_offset); } else { edge_blocks_ptr[i].next = 0; } } }
int tabsort(const char* infn, const char* outfn, const char* colname, int descending) { FILE* fin; FILE* fout; int ext, nextens; off_t start, size; void* data = NULL; int* perm = NULL; unsigned char* map = NULL; size_t mapsize = 0; anqfits_t* anq = NULL; fin = fopen(infn, "rb"); if (!fin) { SYSERROR("Failed to open input file %s", infn); return -1; } fout = fopen(outfn, "wb"); if (!fout) { SYSERROR("Failed to open output file %s", outfn); goto bailout; } // copy the main header exactly. anq = anqfits_open(infn); if (!anq) { ERROR("Failed to open \"%s\"", infn); goto bailout; } start = anqfits_header_start(anq, 0); size = anqfits_header_size (anq, 0); if (pipe_file_offset(fin, start, size, fout)) { ERROR("Failed to copy primary FITS header."); goto bailout; } nextens = anqfits_n_ext(anq); //logverb("Sorting %i extensions.\n", nextens); for (ext=1; ext<nextens; ext++) { int c; qfits_table* table; qfits_col* col; int mgap; off_t mstart; size_t msize; int atomsize; int (*sort_func)(const void*, const void*); unsigned char* tabledata; unsigned char* tablehdr; off_t hdrstart, hdrsize, datsize, datstart; int i; hdrstart = anqfits_header_start(anq, ext); hdrsize = anqfits_header_size (anq, ext); datstart = anqfits_data_start (anq, ext); datsize = anqfits_data_size (anq, ext); if (!anqfits_is_table(anq, ext)) { ERROR("Extension %i isn't a table. Skipping.\n", ext); continue; } table = anqfits_get_table(anq, ext); if (!table) { ERROR("Failed to open table: file %s, extension %i. Skipping.", infn, ext); continue; } c = fits_find_column(table, colname); if (c == -1) { ERROR("Couldn't find column named \"%s\" in extension %i. Skipping.", colname, ext); continue; } col = table->col + c; switch (col->atom_type) { case TFITS_BIN_TYPE_D: data = realloc(data, table->nr * sizeof(double)); if (descending) sort_func = compare_doubles_desc; else sort_func = compare_doubles_asc; break; case TFITS_BIN_TYPE_E: data = realloc(data, table->nr * sizeof(float)); if (descending) sort_func = compare_floats_desc; else sort_func = compare_floats_asc; break; case TFITS_BIN_TYPE_K: data = realloc(data, table->nr * sizeof(int64_t)); if (descending) sort_func = compare_int64_desc; else sort_func = compare_int64_asc; break; default: ERROR("Column %s is neither FITS type D, E, nor K. Skipping.", colname); continue; } // Grab the sort column. atomsize = fits_get_atom_size(col->atom_type); printf("Reading sort column \"%s\"\n", colname); qfits_query_column_seq_to_array(table, c, 0, table->nr, data, atomsize); // Sort it. printf("Sorting sort column\n"); perm = permuted_sort(data, atomsize, sort_func, NULL, table->nr); // mmap the input file. printf("mmapping input file\n"); start = hdrstart; size = hdrsize + datsize; get_mmap_size(start, size, &mstart, &msize, &mgap); mapsize = msize; map = mmap(NULL, mapsize, PROT_READ, MAP_SHARED, fileno(fin), mstart); if (map == MAP_FAILED) { SYSERROR("Failed to mmap input file %s", infn); map = NULL; goto bailout; } tabledata = map + (off_t)mgap + (datstart - hdrstart); tablehdr = map + (off_t)mgap; // Copy the table header without change. printf("Copying table header.\n"); if (fwrite(tablehdr, 1, hdrsize, fout) != hdrsize) { SYSERROR("Failed to write FITS table header"); goto bailout; } for (i=0; i<table->nr; i++) { unsigned char* rowptr; if (i % 100000 == 0) printf("Writing row %i\n", i); rowptr = tabledata + (off_t)(perm[i]) * (off_t)table->tab_w; if (fwrite(rowptr, 1, table->tab_w, fout) != table->tab_w) { SYSERROR("Failed to write FITS table row"); goto bailout; } } munmap(map, mapsize); map = NULL; free(perm); perm = NULL; if (fits_pad_file(fout)) { ERROR("Failed to add padding to extension %i", ext); goto bailout; } qfits_table_close(table); } free(data); if (fclose(fout)) { SYSERROR("Error closing output file"); fout = NULL; goto bailout; } fclose(fin); anqfits_close(anq); printf("Done\n"); return 0; bailout: free(data); free(perm); if (fout) fclose(fout); fclose(fin); if (map) munmap(map, mapsize); return -1; }