END_TEST START_TEST(test_bf_double_close) { bloom_filter_params params = {0, 0, 1e6, 1e-4}; bf_params_for_capacity(¶ms); bloom_bitmap map; bloom_bloomfilter filter; bitmap_from_file(-1, params.bytes, ANONYMOUS, &map); bf_from_bitmap(&map, params.k_num, 1, &filter); fail_unless(bf_close(&filter) == 0); fail_unless(bf_close(&filter) == -1); }
END_TEST START_TEST(test_bf_shared_compatible_persist) { bloom_filter_params params = {0, 0, 1e6, 1e-4}; bf_params_for_capacity(¶ms); bloom_bitmap map; bloom_bloomfilter filter; fail_unless(bitmap_from_filename("/tmp/shared_compat_persist.mmap", params.bytes, 1, PERSISTENT, &map) == 0); fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0); fchmod(map.fileno, 0777); // Check all the keys get added char buf[100]; int res; for (int i=0;i<1000;i++) { snprintf((char*)&buf, 100, "test%d", i); res = bf_add(&filter, (char*)&buf); fail_unless(res == 1); } fail_unless(bf_close(&filter) == 0); // Test all the keys are contained fail_unless(bitmap_from_filename("/tmp/shared_compat_persist.mmap", params.bytes, 1, SHARED, &map) == 0); fail_unless(bf_from_bitmap(&map, params.k_num, 1, &filter) == 0); for (int i=0;i<1000;i++) { snprintf((char*)&buf, 100, "test%d", i); res = bf_contains(&filter, (char*)&buf); fail_unless(res == 1); } unlink("/tmp/shared_compat_persist.mmap"); }
END_TEST START_TEST(test_flush_close) { bloom_filter_params params = {0, 0, 1e6, 1e-4}; bf_params_for_capacity(¶ms); bloom_bitmap map; bloom_bloomfilter filter; bitmap_from_filename("/tmp/test_flush_close.mmap", params.bytes, 1, SHARED, &map); bf_from_bitmap(&map, params.k_num, 1, &filter); fail_unless(bf_flush(&filter) == 0); fail_unless(bf_close(&filter) == 0); unlink("/tmp/test_flush_close.mmap"); }
/** * This beast mode method scans the data directory * belonging to this filter for any existing filters, * and restores the SBF * @return 0 on success. -1 on error. */ static int discover_existing_filters(bloom_filter *f) { // Scan through the folder looking for data files struct dirent **namelist; int num; // Filter only data dirs, in sorted order num = scandir(f->full_path, &namelist, filter_data_files, alphasort); if (num == -1) { syslog(LOG_ERR, "Failed to scan files for filter '%s'. %s", f->filter_name, strerror(errno)); return -1; } syslog(LOG_INFO, "Found %d files for filter %s.", num, f->filter_name); // Speical case when there are no filters if (num == 0) { int res = create_sbf(f, 0, NULL); return res; } // Allocate space for all the filter bloom_bitmap **maps = malloc(num * sizeof(bloom_bitmap*)); bloom_bloomfilter **filters = malloc(num * sizeof(bloom_bloomfilter*)); // Initialize the bitmaps and bloom filters int res; int err = 0; uint64_t size; bitmap_mode mode = (f->config->use_mmap) ? SHARED : PERSISTENT; for (int i=0; i < num && !err; i++) { // Get the full path to the bitmap char *bitmap_path = join_path(f->full_path, namelist[i]->d_name); syslog(LOG_INFO, "Discovered bloom filter: %s.", bitmap_path); // Get the size size = get_size(bitmap_path); if (size == 0) { err = 1; syslog(LOG_ERR, "Failed to get the filesize for: %s. %s", bitmap_path, strerror(errno)); free(bitmap_path); break; } // Create the bitmap bloom_bitmap *bitmap = maps[num - i - 1] = malloc(sizeof(bloom_bitmap)); res = bitmap_from_filename(bitmap_path, size, 0, mode, bitmap); if (res != 0) { err = 1; syslog(LOG_ERR, "Failed to load bitmap for: %s. %s", bitmap_path, strerror(errno)); free(bitmap); free(bitmap_path); break; } // Create the bloom filter bloom_bloomfilter *filter = filters[num - i - 1] = malloc(sizeof(bloom_bloomfilter)); res = bf_from_bitmap(bitmap, 1, 0, filter); if (res != 0) { err = 1; syslog(LOG_ERR, "Failed to load bloom filter for: %s. [%d]", bitmap_path, res); free(filter); bitmap_close(bitmap); free(bitmap); free(bitmap_path); break; } // Cleanup free(bitmap_path); } // Free the memory associated with scandir for (int i=0; i < num; i++) free(namelist[i]); free(namelist); // Return if there was an error if (err) return -1; // Create the SBF res = create_sbf(f, num, filters); // Cleanup on err if (res != 0) { syslog(LOG_ERR, "Failed to make scalable bloom filter for: %s.", f->filter_name); // For f***s sake. We need to clean up so much shit now. for (int i=0; i < num; i++) { bf_close(filters[i]); bitmap_close(maps[i]); free(filters[i]); free(maps[i]); } } // Increase our page ins f->counters.page_ins += 1; // Remove the filters list free(maps); free(filters); return (err) ? -1 : 0; }
int main (int argc, char *argv[]) { int n, ib = 0, nb, flags = 0, has_q = 0; BINARYIO *bf; static char basename[33] = "Base"; if (argc < 3) print_usage (usgmsg, NULL); /* get options */ while ((n = getargs (argc, argv, options)) > 0) { switch (n) { case 'f': flags &= ~OPEN_FORTRAN; flags |= OPEN_ASCII; break; case 'u': flags &= ~OPEN_ASCII; flags |= OPEN_FORTRAN; break; case 's': mblock = 0; break; case 'p': whole = 0; break; case 'i': use_iblank = 1; /* fall through */ case 'n': has_iblank = 1; break; case 'd': is_double = 1; break; case 'M': flags &= ~MACH_UNKNOWN; flags |= get_machine (argarg); break; case 'b': ib = atoi (argarg); break; case 'B': strncpy (basename, argarg, 32); basename[32] = 0; break; case 'g': gamma = atof (argarg); if (gamma <= 1.0) FATAL (NULL, "invalid value for gamma"); break; case 'c': convert = 1; break; } } if (argind > argc - 2) print_usage (usgmsg, "XYZfile and/or CGNSfile not given"); /* read Plot3d file */ printf ("reading PLOT3D grid file %s\n", argv[argind]); printf (" as %s-block %s", mblock ? "multi" : "single", flags == OPEN_ASCII ? "ASCII" : (flags == OPEN_FORTRAN ? "FORTRAN unformatted" : "binary")); if (has_iblank) printf (" with iblank array"); putchar ('\n'); if (!file_exists (argv[argind])) FATAL (NULL, "XYZ file does not exist or is not a file"); if (NULL == (bf = bf_open (argv[argind], flags | OPEN_READ))) { fprintf (stderr, "can't open <%s> for reading", argv[argind]); exit (1); } read_xyz (bf); bf_close (bf); if (use_iblank) build_interfaces (); /* read solution file if given */ if (++argind < argc-1) { printf ("\nreading PLOT3D solution file %s\n", argv[argind]); if (!file_exists (argv[argind])) FATAL (NULL, "Solution file does not exist or is not a file"); if (NULL == (bf = bf_open (argv[argind], flags | OPEN_READ))) { fprintf (stderr, "can't open <%s> for reading", argv[argind]); exit (1); } read_q (bf); bf_close (bf); argind++; has_q = 1; } /* open CGNS file */ printf ("\nwriting CGNS file to %s\n", argv[argind]); nb = open_cgns (argv[argind], 0); if (ib) { if (ib > nb) FATAL (NULL, "specified base index out of range"); if (cg_base_read (cgnsfn, ib, basename, &n, &n)) FATAL (NULL, NULL); } if (cg_base_write (cgnsfn, basename, 3, 3, &cgnsbase) || cg_goto (cgnsfn, cgnsbase, "end") || cg_dataclass_write (CGNS_ENUMV(NormalizedByUnknownDimensional))) FATAL (NULL, NULL); printf (" output to base %d - %s\n", cgnsbase, basename); write_zones (); for (n = 1; n <= nZones; n++) { printf ("writing zone %d ... grid", n); fflush (stdout); write_zone_grid (n); write_zone_interface (n); if (has_q) { printf (", solution"); fflush (stdout); write_zone_solution (n, 1); write_solution_field (n, 1, 0); } puts (" done"); } if (has_q) write_reference (); cg_close (cgnsfn); return 0; }
static void read_xyz (BINARYIO *bf) { int i, k, n, nk, nz, np, nmax; int *indices, *iblank; void *xyz; VERTEX *verts; /* get number of grid blocks */ if (mblock) { bf_getints (bf, 1, &nZones); if (nZones < 1 || nZones > 100000) { fprintf (stderr, "found %d blocks\n", nZones); fprintf (stderr, "file type and/or format is probably incorrect\n"); bf_close (bf); exit (1); } } else nZones = 1; printf ("reading %d grid blocks\n", nZones); /* read indices for grids */ indices = (int *) malloc (3 * nZones * sizeof(int)); if (NULL == indices) FATAL ("read_xyz", "malloc failed for grid indices"); bf_getints (bf, 3 * nZones, indices); /* create zone structures */ Zones = new_zone (nZones); for (nmax = 0, nz = 0; nz < nZones; nz++) { Zones[nz].type = CGNS_ENUMV(Structured); for (np = 1, n = 0; n < 3; n++) { nk = indices[3 * nz + n]; Zones[nz].dim[n] = nk; np *= nk; } Zones[nz].vertflags = 7; Zones[nz].datatype = is_double ? CGNS_ENUMV(RealDouble) : CGNS_ENUMV(RealSingle); Zones[nz].nverts = np; Zones[nz].verts = new_vertex (np); nk = whole ? (int)Zones[nz].nverts : (int)(Zones[nz].dim[0] * Zones[nz].dim[1]); if (nmax < nk) nmax = nk; } free (indices); if (is_double) xyz = (void *) malloc (3 * nmax * sizeof(double)); else xyz = (void *) malloc (3 * nmax * sizeof(float)); if (NULL == xyz) FATAL ("read_xyz", "malloc failed for coordinate working array"); if (has_iblank) { iblank = (int *) malloc (nmax * sizeof(int)); if (NULL == iblank) FATAL ("read_xyz", "malloc failed for iblank array"); } else use_iblank = 0; /* read the grid blocks */ for (nz = 0; nz < nZones; nz++) { printf ("reading block %d grid %dx%dx%d ...", nz+1, (int)Zones[nz].dim[0], (int)Zones[nz].dim[1], (int)Zones[nz].dim[2]); fflush (stdout); if (whole) { np = (int)Zones[nz].nverts; nk = 1; } else { np = (int)(Zones[nz].dim[0] * Zones[nz].dim[1]); nk = (int)Zones[nz].dim[2]; } verts = Zones[nz].verts; for (k = 0; k < nk; k++) { if (is_double) bf_getdoubles (bf, 3 * np, xyz); else bf_getfloats (bf, 3 * np, xyz); if (has_iblank) bf_getints (bf, np, iblank); if (is_double) { for (i = 0, n = 0; n < np; n++, i++) verts[n].x = ((double *)xyz)[i]; for (n = 0; n < np; n++, i++) verts[n].y = ((double *)xyz)[i]; for (n = 0; n < np; n++, i++) verts[n].z = ((double *)xyz)[i]; } else { for (i = 0, n = 0; n < np; n++, i++) verts[n].x = ((float *)xyz)[i]; for (n = 0; n < np; n++, i++) verts[n].y = ((float *)xyz)[i]; for (n = 0; n < np; n++, i++) verts[n].z = ((float *)xyz)[i]; } for (n = 0; n < np; n++, verts++) verts->id = use_iblank ? iblank[n] : 1; } puts (" done"); } free (xyz); if (has_iblank) free (iblank); }