Ejemplo n.º 1
0
// Note: assumes one or more of datablock->bounds or output_sel is local
static uint64_t apply_datablock_to_buffer_nonlocal_selections(
		const ADIOS_VARINFO *raw_varinfo, const ADIOS_TRANSINFO *transinfo,
		adios_datablock *datablock,
        void **output_buffer, const ADIOS_SELECTION *output_sel,
        ADIOS_SELECTION **out_inter_sel, int want_out_inter_sel,
        enum ADIOS_FLAG swap_endianness)
{
	int may_have_intersection = 1;
	uint64_t used_count = 0;

	const ADIOS_SELECTION *global_output_buffer_sel = output_sel;
	const ADIOS_SELECTION *global_datablock_bounds = datablock->bounds;

	// Promote output buffer selection and/or datablock selection to global if needed
	if (!is_global_selection(global_output_buffer_sel))
		global_output_buffer_sel = create_writeblock_bounds(&global_output_buffer_sel->u.block, datablock->timestep, raw_varinfo, transinfo);
	if (!is_global_selection(global_datablock_bounds))
		global_datablock_bounds = create_writeblock_bounds(&global_datablock_bounds->u.block, datablock->timestep, raw_varinfo, transinfo);

    // Compute the intersection explicitly if it is requested, or
    // if we need to allocate a fitting output buffer
    if (want_out_inter_sel || !*output_buffer) {
    	*out_inter_sel = adios_selection_intersect_global(global_datablock_bounds, global_output_buffer_sel);
    	may_have_intersection = (*out_inter_sel ? 1 : 0);
    }

    // We can stop immediately if it is known there is no intersection
    if (may_have_intersection) {
    	// Allocate the output buffer if needed (inter_sel is populated by previous if statement)
    	if (!*output_buffer) {
    		const uint64_t chunk_buffer_size = compute_selection_size_in_bytes(*out_inter_sel, datablock->elem_type, datablock->timestep, raw_varinfo, transinfo);
    		*output_buffer = malloc(chunk_buffer_size);

    		// Refitting the output selection to the intersection region, since we
    		// just allocated a buffer for that smaller region
    		if (global_output_buffer_sel != output_sel)
    			common_read_selection_delete((ADIOS_SELECTION *)global_output_buffer_sel);
    		output_sel = *out_inter_sel;
    		global_output_buffer_sel = *out_inter_sel;
    	}

    	// Perform the actual data patching, now that everything is in the global space
    	used_count = adios_patch_data_to_global(
    			*output_buffer, (uint64_t)0, global_output_buffer_sel,
    			datablock->data, datablock->ragged_offset, global_datablock_bounds,
    			datablock->elem_type, swap_endianness);
    }

	// Clean up
	if (global_output_buffer_sel != output_sel)
		common_read_selection_delete((ADIOS_SELECTION *)global_output_buffer_sel);
	if (global_datablock_bounds != datablock->bounds)
		common_read_selection_delete((ADIOS_SELECTION *)global_datablock_bounds);

	return used_count;
}
Ejemplo n.º 2
0
// Populates data transform information about a given variable into an ADIOS_VARTRANSFORM struct
// Return NULL if failed
ADIOS_VARTRANSFORM *  adios_inq_var_transform(const ADIOS_FILE *fp, const ADIOS_VARINFO *varinfo){
	// Get the global metadata
	ADIOS_TRANSINFO* tinfo = common_read_inq_transinfo(fp, varinfo);
	if (tinfo == NULL)
		return NULL;

	// Get the per-PG metadata
	common_read_inq_trans_blockinfo(fp, varinfo, tinfo);
	if (tinfo->orig_blockinfo == NULL || tinfo->transform_metadatas == NULL)
		return NULL;

	// Load all the metadata into the ADIOS_VARTRANSFORM datastructure
	ADIOS_VARTRANSFORM *vartransform = (ADIOS_VARTRANSFORM*) malloc(sizeof(ADIOS_VARTRANSFORM));
	*vartransform = (ADIOS_VARTRANSFORM){
		.varid = varinfo->varid,
		.sum_nblocks = varinfo->sum_nblocks,
		.transform_type = tinfo->transform_type,
		.should_free_transform_metadata = tinfo->should_free_transform_metadata,
		.transform_metadatas = tinfo->transform_metadatas
	};

	// Transfer ownership of the transform_metadatas array to the new struct, then free the struct
	tinfo->transform_metadatas = NULL;
	common_read_free_transinfo(varinfo, tinfo);

	return vartransform;
}

#define MYFREE(p) {if (p){ free((void*)(p)); (p) = NULL; }}
void adios_free_var_transform(ADIOS_VARTRANSFORM *vartransform) {
	if (vartransform->transform_metadatas) {
		if (vartransform->should_free_transform_metadata) {
			int i;
			for (i = 0; i < vartransform->sum_nblocks; i++)
				MYFREE(vartransform->transform_metadatas[i].content);
		}
		MYFREE(vartransform->transform_metadatas);
	}
	MYFREE(vartransform);
}


void adios_free_pg_intersections(ADIOS_PG_INTERSECTIONS **intersections){
	ADIOS_PG_INTERSECTIONS * intsec = *intersections;
	int i = 0;
	for(i=0; i < intsec->npg; i++){
		ADIOS_PG_INTERSECTION inter = intsec->intersections[i];
		common_read_selection_delete(inter.pg_bounds_sel);
		common_read_selection_delete(inter.intersection_sel);
	}
	intsec->npg = 0;
	intsec->intersections = NULL;
	MYFREE(*intersections);
}
Ejemplo n.º 3
0
static int generate_read_request_for_pg(
		const ADIOS_VARINFO *raw_varinfo, const ADIOS_TRANSINFO *transinfo,
		const ADIOS_SELECTION *sel,
		int timestep, int timestep_blockidx, int blockidx,
		adios_transform_read_request *readreq)
{
    const ADIOS_SELECTION *pg_bounds_sel;
    ADIOS_SELECTION *pg_intersection_sel;
    ADIOS_SELECTION *pg_writeblock_sel;

    const ADIOS_VARBLOCK *raw_vb = &raw_varinfo->blockinfo[blockidx];
    const ADIOS_VARBLOCK *orig_vb = &transinfo->orig_blockinfo[blockidx];

    pg_bounds_sel = create_pg_bounds_from_varblock(transinfo->orig_ndim, orig_vb);
    pg_writeblock_sel = common_read_selection_writeblock(blockidx);
    pg_writeblock_sel->u.block.is_absolute_index = 1;

    // Find the intersection, if any
    if (is_global_selection(sel)) {
    	pg_intersection_sel = adios_selection_intersect_global(pg_bounds_sel, sel);
    } else if (sel->type == ADIOS_SELECTION_WRITEBLOCK) {
    	pg_intersection_sel = adios_selection_intersect_local(pg_writeblock_sel, sel, timestep, raw_varinfo, transinfo);
    } else {
    	abort(); // Should never be called with other types of selections
    }

    // If there is an intersection, generate a corresponding PG read request
    if (pg_intersection_sel) {
        // Make a PG read request group, and fill it with some subrequests, and link it into the read reqgroup
        adios_transform_pg_read_request *new_pg_readreq;
        new_pg_readreq = adios_transform_pg_read_request_new(timestep, timestep_blockidx, blockidx,
                                                              transinfo->orig_ndim, raw_varinfo->ndim,
                                                              orig_vb, raw_vb,
                                                              pg_intersection_sel, pg_bounds_sel,
                                                              transinfo->transform_metadatas[blockidx].content,
                                                              (uint16_t)transinfo->transform_metadatas[blockidx].length);

        adios_transform_generate_read_subrequests(readreq, new_pg_readreq);
        adios_transform_pg_read_request_append(readreq, new_pg_readreq);

        // Don't free pg_bounds_sel or pg_intersection_sel, since they are now
        // held by the adios_transform_pg_read_request struct
        return 1;
    } else {
        // Cleanup
        common_read_selection_delete((ADIOS_SELECTION *)pg_bounds_sel); // OK to delete, because this function only frees the outer struct, not the arrays within
        return 0;
    }
}
Ejemplo n.º 4
0
// One-to-one global patch functions
inline static uint64_t adios_patch_data_bb_to_bb(void *dst, uint64_t dst_ragged_offset, const ADIOS_SELECTION_BOUNDINGBOX_STRUCT *dst_bb,
                                                 void *src, uint64_t src_ragged_offset, const ADIOS_SELECTION_BOUNDINGBOX_STRUCT *src_bb,
                                                 enum ADIOS_DATATYPES datum_type,
                                                 enum ADIOS_FLAG swap_endianness) {

    const int ndim = dst_bb->ndim;
    const ADIOS_SELECTION_BOUNDINGBOX_STRUCT *inter_bb;
    uint64_t *inter_off_relative_to_dst;
    uint64_t *inter_off_relative_to_src;
    uint64_t volume;

    // Intersect the two bounding boxes
    ADIOS_SELECTION *inter_sel = adios_selection_intersect_bb_bb(dst_bb, src_bb);

    // If there is no intersection, stop now, nothing to do
    if (!inter_sel)
        return 0;

    // (this is guaranteed by the selection intersection code; this is just to check for bugs)
    assert(inter_sel->type == ADIOS_SELECTION_BOUNDINGBOX);
    inter_bb = &inter_sel->u.bb;

    // Compute the offset of the intersection bounding box within each of
    // the source and destination bounding boxes
    assert(dst_bb->ndim == src_bb->ndim);
    inter_off_relative_to_dst = malloc(ndim * sizeof(uint64_t));
    inter_off_relative_to_src = malloc(ndim * sizeof(uint64_t));
    vector_sub(ndim, inter_off_relative_to_dst, inter_bb->start, dst_bb->start);
    vector_sub(ndim, inter_off_relative_to_src, inter_bb->start, src_bb->start);

    // Perform a subvolume memcpy
    copy_subvolume_ragged_offset(
        dst, src, dst_bb->ndim, inter_bb->count,
        dst_bb->count, inter_off_relative_to_dst, dst_ragged_offset,
        src_bb->count, inter_off_relative_to_src, src_ragged_offset,
        datum_type, swap_endianness);

    // Compute the number of elements copied
    volume = compute_volume(ndim, inter_bb->count);

    // Cleanup
    free(inter_off_relative_to_dst);
    free(inter_off_relative_to_src);
    common_read_selection_delete(inter_sel);

    return volume;
}
Ejemplo n.º 5
0
static uint64_t apply_datablock_to_buffer_local_selections(
		const ADIOS_VARINFO *raw_varinfo, const ADIOS_TRANSINFO *transinfo,
		adios_datablock *datablock,
        void **output_buffer, const ADIOS_SELECTION *output_sel,
        ADIOS_SELECTION **out_inter_sel, int want_out_inter_sel,
        enum ADIOS_FLAG swap_endianness)
{
	int may_have_intersection = 1;

	// For writeblock selections, we can use adios_patch_data_to_local,
	// but first we must determine the bounding box of the writeblock selection
    const ADIOS_SELECTION *vb_bounds_sel = create_writeblock_bounds(&output_sel->u.block, datablock->timestep, raw_varinfo, transinfo);

    // Compute the intersection explicitly if it is requested, or
    // if we need to allocate a fitting output buffer
    if (want_out_inter_sel || !*output_buffer) {
        *out_inter_sel = adios_selection_intersect_local(datablock->bounds, output_sel, datablock->timestep, raw_varinfo, transinfo);
        may_have_intersection = (*out_inter_sel ? 1 : 0);
    }

    // Allocate the output buffer if needed (inter_sel is populated by previous if statement)
    if (!*output_buffer) {
    	const uint64_t chunk_buffer_size = compute_selection_size_in_bytes(*out_inter_sel, datablock->elem_type, datablock->timestep, raw_varinfo, transinfo);
		*output_buffer = malloc(chunk_buffer_size);

		// Refitting the output selection to the intersection region, since we
		// just allocated a buffer for that smaller region
		output_sel = *out_inter_sel;
    }

	// Invoke adios_patch_data_to_local to perform the actual patching
	const uint64_t used_count =
			adios_patch_data_to_local(
				*output_buffer, (uint64_t)0, output_sel,
				datablock->data, datablock->ragged_offset, datablock->bounds,
				&vb_bounds_sel->u.bb,
				datablock->elem_type, swap_endianness);

	// Clean up
	common_read_selection_delete((ADIOS_SELECTION *)vb_bounds_sel);
	return used_count;
}
Ejemplo n.º 6
0
void common_query_free(ADIOS_QUERY* q)
{
  if (q == NULL) {
    return;
  }

  if (q->deleteSelectionWhenFreed) {
    common_read_selection_delete(q->sel);
  }

  // Only call a specialized free method if this query has been evaluated using
  // a particular query engine! Otherwise, if a query is created but never
  // evaluated, this will cause segfaults since no query method initialized the query
  if (q->method != ADIOS_QUERY_METHOD_UNKNOWN) {
      assert(q->method < ADIOS_QUERY_METHOD_COUNT);
      if (query_hooks[q->method].adios_query_free_fn != NULL) {
	query_hooks[q->method].adios_query_free_fn(q);
      }
  }

  freeQuery(q);
}
Ejemplo n.º 7
0
void adios_selection_delete (ADIOS_SELECTION *sel)
{
    common_read_selection_delete (sel);
}
Ejemplo n.º 8
0
/*
 * Takes a datablock and applies its data to a given output buffer (described
 * by a given output buffer selection), then frees the given datablock.
 *
 * If *output_buffer is non-NULL, copies the pertinent data from datablock->data
 * into *output_buffer, assuming *output_buffer is shaped like output_sel.
 *
 * If *output_buffer is NULL, allocates a minimum-size buffer to contain the
 * data for the *intersection* of datablock->bounds and output_sel and returns
 * it via *output_buffer.
 *
 * If out_inter_sel != NULL, returns a selection representing the intersection
 * of datablock->bounds and output_sel (i.e., the portion of data that was
 * actually copied) via *out_inter_sel. Otherwise, leaves this argument untouched.
 *
 * This function can handle any combination of datablock bounding selection and
 * output buffer bounding selection:
 * - If both datablock and buffer selections are global (BB, points), performs
 *   a straightforward data patching based on the geometry
 * - If both datablock and buffer selections are local (writeblock), performs
 *   a straghtforward memcpy as appropriate
 * - If the buffer is global but the datablock is local, promotes the datablock
 *   to a bounding box using blockinfo (note: the variable is guaranteed to be
 *   a global array in this case because the user supplied a global selection)
 * - If the buffer is local but the datablock is global, promotes the buffer
 *   to a bounding box using blockinfo (note: the variable may or may not be
 *   a global array in this case; however, even if it is not, both the datablock
 *   and the buffer will be constructed relative to (0,0,...,0), so it will do
 *   the right thing).
 *
 * @return the number of elements patched into the output buffer (0 indicates
 *         no data in the given datablock was pertinent to the given output
 *         buffer selection)
 */
static uint64_t apply_datablock_to_buffer_and_free(
		const ADIOS_VARINFO *raw_varinfo, const ADIOS_TRANSINFO *transinfo,
		adios_datablock *datablock,
        void **output_buffer, const ADIOS_SELECTION *output_sel, ADIOS_SELECTION **out_inter_sel, enum ADIOS_FLAG swap_endianness)
{
    uint64_t used_count = 0;
    ADIOS_SELECTION *inter_sel = NULL;
    assert(raw_varinfo && transinfo && datablock && output_buffer && output_sel);

    // Check preconditions
    if (datablock->bounds->type != ADIOS_SELECTION_BOUNDINGBOX &&
    	datablock->bounds->type != ADIOS_SELECTION_POINTS &&
    	datablock->bounds->type != ADIOS_SELECTION_WRITEBLOCK)
    {
        adios_error(err_operation_not_supported,
                    "Only results of bounding box, points, or writeblock selection types "
                    "are currently accepted from transform plugins (received selection type %d)\n",
                    datablock->bounds->type);
        return 0;
    }
    if (output_sel->type != ADIOS_SELECTION_BOUNDINGBOX &&
    	output_sel->type != ADIOS_SELECTION_POINTS &&
    	output_sel->type != ADIOS_SELECTION_WRITEBLOCK)
    {
        adios_error_at_line(err_operation_not_supported, __FILE__, __LINE__,
                            "Internal error: only bounding box, points, or writeblock selection types "
                            "are currently supported in apply_datablock_to_buffer_and_free (received selection type %d)\n",
                            datablock->bounds->type);
        return 0;
    }

    // Invoke the appropriate helper function depending
    // on whether at least one of datablock->bounds or
    // output_sel is global
    if (!is_global_selection(datablock->bounds) && !is_global_selection(output_sel)) {
    	used_count = apply_datablock_to_buffer_local_selections(
    			raw_varinfo, transinfo,
    			datablock, output_buffer, output_sel,
    			&inter_sel, (out_inter_sel ? 1 : 0),
    			swap_endianness);
    } else {
    	used_count = apply_datablock_to_buffer_nonlocal_selections(
    			raw_varinfo, transinfo,
    			datablock, output_buffer, output_sel,
    			&inter_sel, (out_inter_sel ? 1 : 0),
    			swap_endianness);
    }

    // Clean up the returned intersection if it is not wanted by the caller
	if (inter_sel) {
		if (out_inter_sel) {
			*out_inter_sel = inter_sel;
		} else {
			// TODO: Deep delete the selection (delete points list, start/count arrays, etc.)
			common_read_selection_delete(inter_sel);
		}
	}

    // Free the datablock
    adios_datablock_free(&datablock, 1);
    return used_count;
}
Ejemplo n.º 9
0
int common_query_evaluate(ADIOS_QUERY* q, 
			  ADIOS_SELECTION* outputBoundary, 
			  int timeStep, 
			  uint64_t batchSize, // limited by maxResult
			  ADIOS_SELECTION** result)
{  
	double start = 0, end = 0;
#ifdef BREAKDOWN
	start = dclock();
#endif
  if (q == 0) {
    log_debug("Error: empty query will not be evaluated!");
    return -1;
  }
    int actualTimeStep = adios_check_query_at_timestep(q, timeStep);
    if (actualTimeStep == -1) {
      return -1;
    }

    /*
    if (checkCompatibility(q) != 0) {
      log_error("query components are not compatible at this time step.\n");
      return -1;
    }
    */
    /*
    if ((q->onTimeStep >= 0) && (q->onTimeStep != timeStep)) {
        int updateResult = updateBlockSizeIfNeeded(q, timeStep);
        if (updateResult < 0) {
            log_error("Error with this timestep %d. Can not proceed. \n", timeStep);
            return -1;
        }
        if (updateResult > 0) { // blocks were updated. check compatitibity
            if (checkCompatibility(q) <= 0) {
                return -1;
            }
        }
    }
    */
    int freeOutputBoundary = 0;
    if ((outputBoundary != NULL) && (outputBoundary->type == ADIOS_SELECTION_WRITEBLOCK)) {
        outputBoundary = convertWriteblockToBoundingBox(q, &outputBoundary->u.block, timeStep);
        if (!outputBoundary) {
	  adios_error(err_invalid_argument,
		      "Attempt to use writeblock output selection on a query where not "
		      "all variables participating have the same varblock bounding box "
		      "at that writeblock index (index = %d)\n",
		      outputBoundary->u.block.index);
	  return -1;
        }
        freeOutputBoundary = 1;
    }

    enum ADIOS_QUERY_METHOD m = detect_and_set_query_method (q);

    if (query_hooks[m].adios_query_evaluate_fn != NULL) {
      int retval = query_hooks[m].adios_query_evaluate_fn(q, timeStep, batchSize, outputBoundary, result);	      
      if (freeOutputBoundary) common_read_selection_delete(outputBoundary);

#ifdef BREAKDOWN
      end = dclock();
      printf("total time [frame + plugin + adios] : %f \n", end - start);
#endif
      return retval;
    } 
    log_debug ("No selection method is supported for method: %d\n", m);

#ifdef BREAKDOWN
      end = dclock();
      printf("total time [frame + plugin + adios] : %f \n", end - start);
#endif

    return -1;
}
Ejemplo n.º 10
0
ADIOS_PG_INTERSECTIONS * adios_find_intersecting_pgs(const ADIOS_FILE *fp, int varid, const ADIOS_SELECTION *sel, const int from_step, const int nsteps) {
    // Declares
    adios_transform_read_request *new_reqgroup;
    int blockidx, timestep, timestep_blockidx;
    int curblocks, start_blockidx, end_blockidx;
    int intersects;
    ADIOS_VARBLOCK *raw_vb, *vb;

    enum ADIOS_FLAG swap_endianness = (fp->endianness == get_system_endianness()) ? adios_flag_no : adios_flag_yes;
    int to_steps = from_step + nsteps;

    // As long as we don't free/destroy it, using the infocache from the file will have no effect on future
    // operations using the file (except possibly speeding them up, so "constness" is still respected
    adios_infocache *infocache = common_read_get_file_infocache((ADIOS_FILE*)fp);

    ADIOS_PG_INTERSECTIONS *resulting_intersections = (ADIOS_PG_INTERSECTIONS *)calloc(1, sizeof(ADIOS_PG_INTERSECTIONS));
    resulting_intersections->npg = 0;

    int intersection_capacity = INITIAL_INTERSECTION_CAPACITY;
    resulting_intersections->intersections = (ADIOS_PG_INTERSECTION *)calloc(intersection_capacity, sizeof(ADIOS_PG_INTERSECTION));

    // Precondition checking
    if (sel->type != ADIOS_SELECTION_BOUNDINGBOX &&
        sel->type != ADIOS_SELECTION_POINTS) {
        adios_error(err_operation_not_supported, "Only bounding box and point selections are currently supported during read on transformed variables.");
    }

    // Still respecting constness, since we're going to undo this
    const data_view_t old_view = adios_read_set_data_view((ADIOS_FILE*)fp, LOGICAL_DATA_VIEW); // Temporarily go to logical data view

    const ADIOS_VARINFO *varinfo = adios_infocache_inq_varinfo(fp, infocache, varid);
    assert(from_step >= 0 && to_steps <= varinfo->nsteps);

    // Compute the blockidx range, given the timesteps
    compute_blockidx_range(varinfo, from_step, to_steps, &start_blockidx, &end_blockidx);

    // Retrieve blockinfos, if they haven't been done retrieved
    if (!varinfo->blockinfo)
    	common_read_inq_var_blockinfo(fp, (ADIOS_VARINFO *)varinfo);

    // Undoing view set (returning to const state)
    adios_read_set_data_view((ADIOS_FILE*)fp, old_view); // Reset the data view to whatever it was before

    // Assemble read requests for each varblock
    blockidx = start_blockidx;
    timestep = from_step;
    timestep_blockidx = 0;

    while (blockidx != end_blockidx) { //for (blockidx = startblock_idx; blockidx != endblock_idx; blockidx++) {
        ADIOS_SELECTION *pg_bounds_sel;
        ADIOS_SELECTION *pg_intersection_sel;

        vb = &varinfo->blockinfo[blockidx];
        pg_bounds_sel = create_pg_bounds(varinfo->ndim, vb);

        // Find the intersection, if any
        pg_intersection_sel = adios_selection_intersect_global(pg_bounds_sel, sel);
        if (pg_intersection_sel) {
        	// Expand the PG intersection array, if needed
        	if (resulting_intersections->npg == intersection_capacity) {
        		intersection_capacity *= 2;
        		resulting_intersections->intersections = (ADIOS_PG_INTERSECTION *)realloc(resulting_intersections->intersections, intersection_capacity * sizeof(ADIOS_PG_INTERSECTION));

        		if (!resulting_intersections->intersections) {
        			adios_error (err_no_memory, "Cannot allocate buffer for PG intersection results in adios_find_intersecting_pgs (required %llu bytes)\n", intersection_capacity * sizeof(ADIOS_PG_INTERSECTION));
        			return NULL;
        		}
        	}

        	ADIOS_PG_INTERSECTION *intersection = &resulting_intersections->intersections[resulting_intersections->npg];
        	intersection->timestep = timestep;
        	intersection->blockidx = blockidx;
        	intersection->blockidx_in_timestep = timestep_blockidx;
        	intersection->intersection_sel = pg_intersection_sel;
        	intersection->pg_bounds_sel = pg_bounds_sel;

        	resulting_intersections->npg++;
        } else {
            // Cleanup
            common_read_selection_delete(pg_bounds_sel); // OK to delete, because this function only frees the outer struct, not the arrays within
        }

        // Increment block indexes
        blockidx++;
        timestep_blockidx++;
        if (timestep_blockidx == varinfo->nblocks[timestep]) {
            timestep_blockidx = 0;
            timestep++;
        }
    }

    return resulting_intersections;
}