Exemple #1
0
// The if statements impose a total order on the selection types, and call this function
// with arguments swapped if they are out of this order. This simplifies the above helper
// functions.
ADIOS_SELECTION * adios_selection_intersect_global(const ADIOS_SELECTION *s1, const ADIOS_SELECTION *s2) {
	if (!is_global_selection(s1) || !is_global_selection(s2)) {
    	adios_error_at_line(err_invalid_argument, __FILE__, __LINE__, "Internal error: adios_selection_intersect_global called on non-global selection(s)");
    	return NULL;
	}

	switch (s1->type) {
    case ADIOS_SELECTION_BOUNDINGBOX:
    {
        const ADIOS_SELECTION_BOUNDINGBOX_STRUCT *bb1 = &s1->u.bb;
        return adios_selection_intersect_bb(bb1, s2);
    }
    case ADIOS_SELECTION_POINTS:
    {
        const ADIOS_SELECTION_POINTS_STRUCT *pts1 = &s1->u.points;
        if (s1->type == ADIOS_SELECTION_BOUNDINGBOX) {
            return adios_selection_intersect_global(s2, s1);
        } else {
            return adios_selection_intersect_pts(pts1, s2);
        }
    }
    default:
        adios_error_at_line(err_invalid_argument, __FILE__, __LINE__, "Unknown selection type %d", s1->type);
        return NULL;
    }
}
Exemple #2
0
uint64_t adios_patch_data_to_global(void *dst, uint64_t dst_ragged_offset, const ADIOS_SELECTION *dst_sel,
                                    void *src, uint64_t src_ragged_offset, const ADIOS_SELECTION *src_sel,
                                    enum ADIOS_DATATYPES datum_type, enum ADIOS_FLAG swap_endianness)
{
	if (!is_global_selection(dst_sel) || !is_global_selection(src_sel)) {
    	adios_error_at_line(err_invalid_argument, __FILE__, __LINE__, "Internal error: adios_patch_data_to_global called on non-global selection type(s)");
    	return 0;
	}

	switch (dst_sel->type) {
    case ADIOS_SELECTION_BOUNDINGBOX:
    {
        const ADIOS_SELECTION_BOUNDINGBOX_STRUCT *dst_bb = &dst_sel->u.bb;
        return adios_patch_data_to_bb(dst, dst_ragged_offset, dst_bb, src, src_ragged_offset, src_sel, datum_type, swap_endianness);
    }
    case ADIOS_SELECTION_POINTS:
    {
        const ADIOS_SELECTION_POINTS_STRUCT *dst_pts = &dst_sel->u.points;
        return adios_patch_data_to_pts(dst, dst_ragged_offset, dst_pts, src, src_ragged_offset, src_sel, datum_type, swap_endianness);
    }
    default:
        adios_error_at_line(err_invalid_argument, __FILE__, __LINE__, "Unknown selection type %d", dst_sel->type);
        return 0;
    }
}
// Note: assumes one or more of datablock->bounds or output_sel is local
static uint64_t apply_datablock_to_buffer_nonlocal_selections(
		const ADIOS_VARINFO *raw_varinfo, const ADIOS_TRANSINFO *transinfo,
		adios_datablock *datablock,
        void **output_buffer, const ADIOS_SELECTION *output_sel,
        ADIOS_SELECTION **out_inter_sel, int want_out_inter_sel,
        enum ADIOS_FLAG swap_endianness)
{
	int may_have_intersection = 1;
	uint64_t used_count = 0;

	const ADIOS_SELECTION *global_output_buffer_sel = output_sel;
	const ADIOS_SELECTION *global_datablock_bounds = datablock->bounds;

	// Promote output buffer selection and/or datablock selection to global if needed
	if (!is_global_selection(global_output_buffer_sel))
		global_output_buffer_sel = create_writeblock_bounds(&global_output_buffer_sel->u.block, datablock->timestep, raw_varinfo, transinfo);
	if (!is_global_selection(global_datablock_bounds))
		global_datablock_bounds = create_writeblock_bounds(&global_datablock_bounds->u.block, datablock->timestep, raw_varinfo, transinfo);

    // Compute the intersection explicitly if it is requested, or
    // if we need to allocate a fitting output buffer
    if (want_out_inter_sel || !*output_buffer) {
    	*out_inter_sel = adios_selection_intersect_global(global_datablock_bounds, global_output_buffer_sel);
    	may_have_intersection = (*out_inter_sel ? 1 : 0);
    }

    // We can stop immediately if it is known there is no intersection
    if (may_have_intersection) {
    	// Allocate the output buffer if needed (inter_sel is populated by previous if statement)
    	if (!*output_buffer) {
    		const uint64_t chunk_buffer_size = compute_selection_size_in_bytes(*out_inter_sel, datablock->elem_type, datablock->timestep, raw_varinfo, transinfo);
    		*output_buffer = malloc(chunk_buffer_size);

    		// Refitting the output selection to the intersection region, since we
    		// just allocated a buffer for that smaller region
    		if (global_output_buffer_sel != output_sel)
    			a2sel_free((ADIOS_SELECTION *)global_output_buffer_sel);
    		output_sel = *out_inter_sel;
    		global_output_buffer_sel = *out_inter_sel;
    	}

    	// Perform the actual data patching, now that everything is in the global space
    	used_count = adios_patch_data_to_global(
    			*output_buffer, (uint64_t)0, global_output_buffer_sel,
    			datablock->data, datablock->ragged_offset, global_datablock_bounds,
    			datablock->elem_type, swap_endianness);
    }

	// Clean up
	if (global_output_buffer_sel != output_sel)
		a2sel_free((ADIOS_SELECTION *)global_output_buffer_sel);
	if (global_datablock_bounds != datablock->bounds)
		a2sel_free((ADIOS_SELECTION *)global_datablock_bounds);

	return used_count;
}
static int generate_read_request_for_pg(
		const ADIOS_VARINFO *raw_varinfo, const ADIOS_TRANSINFO *transinfo,
		const ADIOS_SELECTION *sel,
		int timestep, int timestep_blockidx, int blockidx,
		adios_transform_read_request *readreq)
{
    const ADIOS_SELECTION *pg_bounds_sel;
    ADIOS_SELECTION *pg_intersection_sel;
    ADIOS_SELECTION *pg_writeblock_sel;

    const ADIOS_VARBLOCK *raw_vb = &raw_varinfo->blockinfo[blockidx];
    const ADIOS_VARBLOCK *orig_vb = &transinfo->orig_blockinfo[blockidx];

    pg_bounds_sel = create_pg_bounds_from_varblock(transinfo->orig_ndim, orig_vb);
    pg_writeblock_sel = a2sel_writeblock(blockidx);
    pg_writeblock_sel->u.block.is_absolute_index = 1;

    // Find the intersection, if any
    if (is_global_selection(sel)) {
    	pg_intersection_sel = adios_selection_intersect_global(pg_bounds_sel, sel);
    } else if (sel->type == ADIOS_SELECTION_WRITEBLOCK) {
    	pg_intersection_sel = adios_selection_intersect_local(pg_writeblock_sel, sel, timestep, raw_varinfo, transinfo);
    } else {
    	abort(); // Should never be called with other types of selections
    }
    a2sel_free(pg_writeblock_sel);

    // If there is an intersection, generate a corresponding PG read request
    if (pg_intersection_sel) {
        // Make a PG read request group, and fill it with some subrequests, and link it into the read reqgroup
        adios_transform_pg_read_request *new_pg_readreq;
        new_pg_readreq = adios_transform_pg_read_request_new(timestep, timestep_blockidx, blockidx,
                                                              transinfo->orig_ndim, raw_varinfo->ndim,
                                                              orig_vb, raw_vb,
                                                              pg_intersection_sel, pg_bounds_sel,
                                                              transinfo->transform_metadatas[blockidx].content,
                                                              (uint16_t)transinfo->transform_metadatas[blockidx].length);

        adios_transform_generate_read_subrequests(readreq, new_pg_readreq);
        adios_transform_pg_read_request_append(readreq, new_pg_readreq);

        // Don't free pg_bounds_sel or pg_intersection_sel, since they are now
        // held by the adios_transform_pg_read_request struct
        return 1;
    } else {
        // Cleanup
        a2sel_free((ADIOS_SELECTION *)pg_bounds_sel); // OK to delete, because this function only frees the outer struct, not the arrays within
        return 0;
    }
}
Exemple #5
0
//
// Any-on-any local intersection function
//
ADIOS_SELECTION * adios_selection_intersect_local(const ADIOS_SELECTION *s1, const ADIOS_SELECTION *s2, int timestep, const ADIOS_VARINFO *raw_varinfo, const ADIOS_TRANSINFO *transinfo)
{
	if (is_global_selection(s1) || is_global_selection(s2)) {
    	adios_error_at_line(err_invalid_argument, __FILE__, __LINE__, "Internal error: adios_selection_intersect_local called on non-local selection(s)");
    	return NULL;
	}

	switch (s1->type) {
    case ADIOS_SELECTION_WRITEBLOCK:
    {
        const ADIOS_SELECTION_WRITEBLOCK_STRUCT *wb1 = &s1->u.block;
        return adios_selection_intersect_wb(wb1, s2, timestep, raw_varinfo, transinfo);
    }
    case ADIOS_SELECTION_AUTO:
    {
    	adios_error_at_line(err_invalid_argument, __FILE__, __LINE__, "Unsupported selection type AUTO in adios_selection_intersect_local");
    	return NULL;
    }
    default:
        adios_error_at_line(err_invalid_argument, __FILE__, __LINE__, "Unknown selection type %d", s1->type);
        return NULL;
    }
}
/*
 * Takes a datablock and applies its data to the user buffer for the given
 * read request group, then frees the given datablock. Assumes there is, in
 * fact, a user buffer (i.e., it is not NULL).
 *
 * Assumes that the datablock selection is of type bounding box.
 *
 * NOTE: also frees the data buffer within the datablock
 *
 * @return non-zero if some data in the datablock intersected the read
 *         request's selection, and was applied; returns 0 otherwise.
 */
static int apply_datablock_to_result_and_free(adios_datablock *datablock,
                                              adios_transform_read_request *reqgroup)
{
    assert(datablock); assert(reqgroup);
    assert(reqgroup->orig_sel);
    assert(reqgroup->orig_data);

    void *output_buffer;
    if (is_global_selection(reqgroup->orig_sel)) {
    	// All timestep results have the same size in bytes, so offset the
    	// output pointer by however many timesteps precede this timestep
    	const int timestep_within_request = datablock->timestep - reqgroup->from_steps;
		output_buffer = (char*)reqgroup->orig_data + timestep_within_request * reqgroup->orig_sel_timestep_size;
    } else if (reqgroup->orig_sel->type == ADIOS_SELECTION_WRITEBLOCK) {
    	// For writeblock selections, computing the output buffer position for
    	// the current timestep is a bit trickier, since varblocks may be
    	// different sizes in different timesteps
    	const ADIOS_SELECTION_WRITEBLOCK_STRUCT *orig_sel_wb = &reqgroup->orig_sel->u.block;

    	// Compute the offset into the output buffer at which the current timestep's output should go
    	// For absolute writeblocks, it's always 0, since there is only 1 timestep involved. For
    	// timestep-relative writeblock selections, we add the size of the writeblock (i.e. varblock)
    	// for all previous timesteps in the user's read request.
    	uint64_t output_buffer_offset = 0;
    	if (!orig_sel_wb->is_absolute_index) {
    		int timestep;
    		for (timestep = reqgroup->from_steps; timestep < datablock->timestep; timestep++) {
        		// Compute the size of the writeblock at this timestep and add it to our offset
        		output_buffer_offset += compute_selection_size_in_bytes(reqgroup->orig_sel, reqgroup->transinfo->orig_type, timestep, reqgroup->raw_varinfo, reqgroup->transinfo);
        	}
    	}

    	// Offset the output pointer by the computed amount
    	output_buffer = (char*)reqgroup->orig_data + output_buffer_offset;
    } else {
    	adios_error_at_line(err_unspecified, __FILE__, __LINE__, "Internal error: unexpected selection type %d, this should not be possible here\n", reqgroup->orig_sel->type);
    }

    // Now that we have the output buffer pointer, actually perform the patch operation
    const uint64_t used_count =
            apply_datablock_to_buffer_and_free(
                    reqgroup->raw_varinfo, reqgroup->transinfo,
                    datablock,
                    &output_buffer, reqgroup->orig_sel, NULL /* don't need the intersection */,
                    reqgroup->swap_endianness);

    return used_count != 0;
}
adios_transform_read_request * adios_transform_generate_read_reqgroup(const ADIOS_VARINFO *raw_varinfo, const ADIOS_TRANSINFO* transinfo, const ADIOS_FILE *fp,
                                                                      const ADIOS_SELECTION *sel, int from_steps, int nsteps, const char *param, void *data) {
    // Declares
    adios_transform_read_request *new_readreq;
    int blockidx, timestep, timestep_blockidx;
    int start_blockidx, end_blockidx;

    enum ADIOS_FLAG swap_endianness = (fp->endianness == get_system_endianness()) ? adios_flag_no : adios_flag_yes;

    // In streaming mode, ignore the user's from_steps/nsteps arguments
    if (fp->is_streaming) {
    	from_steps = 0;
    	nsteps = 1;
    }

    // Precondition checking
    assert(is_transform_type_valid(transinfo->transform_type));
    assert(from_steps >= 0 && from_steps + nsteps <= raw_varinfo->nsteps);

    if (sel->type != ADIOS_SELECTION_BOUNDINGBOX &&
        sel->type != ADIOS_SELECTION_POINTS &&
        sel->type != ADIOS_SELECTION_WRITEBLOCK) {
        adios_error(err_operation_not_supported, "Only bounding box, point , and writeblock selections are currently supported for reads on transformed variables.");
    }

    // Retrieve blockinfos, if they haven't been done retrieved
    if (!raw_varinfo->blockinfo)
        common_read_inq_var_blockinfo_raw(fp, (ADIOS_VARINFO*)raw_varinfo);
    if (!transinfo->orig_blockinfo)
        common_read_inq_trans_blockinfo(fp, raw_varinfo, (ADIOS_TRANSINFO*)transinfo);

    // Allocate a new, empty request group
    new_readreq = adios_transform_read_request_new(fp, raw_varinfo, transinfo, sel, from_steps, nsteps, param, data, swap_endianness);

    if (is_global_selection(sel)) {
    	populate_read_request_for_global_selection(raw_varinfo, transinfo, sel, from_steps, nsteps, new_readreq);
    } else {
    	populate_read_request_for_local_selection(raw_varinfo, transinfo, sel, from_steps, nsteps, new_readreq);
    }

    // If this read request does not intersect any PGs, then clear the new read request and return NULL
    if (new_readreq->num_pg_reqgroups == 0) {
        adios_transform_read_request_free(&new_readreq);
        new_readreq = NULL;
    }

    return new_readreq;
}
Exemple #8
0
// NOTE: vb_bounds_sel is the bounding box of the varblock to which dst_sel (a writeblock selection) corresponds
uint64_t adios_patch_data_to_local(void *dst, uint64_t dst_ragged_offset, const ADIOS_SELECTION *dst_sel,
                                   void *src, uint64_t src_ragged_offset, const ADIOS_SELECTION *src_sel,
                                   const ADIOS_SELECTION_BOUNDINGBOX_STRUCT *vb_bounds,
                                   enum ADIOS_DATATYPES datum_type, enum ADIOS_FLAG swap_endianness)
{
	if (is_global_selection(dst_sel)) {
    	adios_error_at_line(err_invalid_argument, __FILE__, __LINE__, "Internal error: adios_patch_data_to_local called on non-global destination selection type %d", dst_sel->type);
    	return 0;
	}

    switch (dst_sel->type) {
    case ADIOS_SELECTION_WRITEBLOCK:
    {
        const ADIOS_SELECTION_WRITEBLOCK_STRUCT *dst_wb = &dst_sel->u.block;
        return adios_patch_data_to_wb(dst, dst_ragged_offset, dst_wb, src, src_ragged_offset, src_sel, vb_bounds, datum_type, swap_endianness);
    }
    default:
        adios_error_at_line(err_invalid_argument, __FILE__, __LINE__, "Unknown selection type %d", dst_sel->type);
        return 0;
    }
}
/*
 * Takes a datablock and applies its data to a given output buffer (described
 * by a given output buffer selection), then frees the given datablock.
 *
 * If *output_buffer is non-NULL, copies the pertinent data from datablock->data
 * into *output_buffer, assuming *output_buffer is shaped like output_sel.
 *
 * If *output_buffer is NULL, allocates a minimum-size buffer to contain the
 * data for the *intersection* of datablock->bounds and output_sel and returns
 * it via *output_buffer.
 *
 * If out_inter_sel != NULL, returns a selection representing the intersection
 * of datablock->bounds and output_sel (i.e., the portion of data that was
 * actually copied) via *out_inter_sel. Otherwise, leaves this argument untouched.
 *
 * This function can handle any combination of datablock bounding selection and
 * output buffer bounding selection:
 * - If both datablock and buffer selections are global (BB, points), performs
 *   a straightforward data patching based on the geometry
 * - If both datablock and buffer selections are local (writeblock), performs
 *   a straghtforward memcpy as appropriate
 * - If the buffer is global but the datablock is local, promotes the datablock
 *   to a bounding box using blockinfo (note: the variable is guaranteed to be
 *   a global array in this case because the user supplied a global selection)
 * - If the buffer is local but the datablock is global, promotes the buffer
 *   to a bounding box using blockinfo (note: the variable may or may not be
 *   a global array in this case; however, even if it is not, both the datablock
 *   and the buffer will be constructed relative to (0,0,...,0), so it will do
 *   the right thing).
 *
 * @return the number of elements patched into the output buffer (0 indicates
 *         no data in the given datablock was pertinent to the given output
 *         buffer selection)
 */
static uint64_t apply_datablock_to_buffer_and_free(
		const ADIOS_VARINFO *raw_varinfo, const ADIOS_TRANSINFO *transinfo,
		adios_datablock *datablock,
        void **output_buffer, const ADIOS_SELECTION *output_sel, ADIOS_SELECTION **out_inter_sel, enum ADIOS_FLAG swap_endianness)
{
    uint64_t used_count = 0;
    ADIOS_SELECTION *inter_sel = NULL;
    assert(raw_varinfo && transinfo && datablock && output_buffer && output_sel);

    // Check preconditions
    if (datablock->bounds->type != ADIOS_SELECTION_BOUNDINGBOX &&
    	datablock->bounds->type != ADIOS_SELECTION_POINTS &&
    	datablock->bounds->type != ADIOS_SELECTION_WRITEBLOCK)
    {
        adios_error(err_operation_not_supported,
                    "Only results of bounding box, points, or writeblock selection types "
                    "are currently accepted from transform plugins (received selection type %d)\n",
                    datablock->bounds->type);
        return 0;
    }
    if (output_sel->type != ADIOS_SELECTION_BOUNDINGBOX &&
    	output_sel->type != ADIOS_SELECTION_POINTS &&
    	output_sel->type != ADIOS_SELECTION_WRITEBLOCK)
    {
        adios_error_at_line(err_operation_not_supported, __FILE__, __LINE__,
                            "Internal error: only bounding box, points, or writeblock selection types "
                            "are currently supported in apply_datablock_to_buffer_and_free (received selection type %d)\n",
                            datablock->bounds->type);
        return 0;
    }

    // Invoke the appropriate helper function depending
    // on whether at least one of datablock->bounds or
    // output_sel is global
    if (!is_global_selection(datablock->bounds) && !is_global_selection(output_sel)) {
    	used_count = apply_datablock_to_buffer_local_selections(
    			raw_varinfo, transinfo,
    			datablock, output_buffer, output_sel,
    			&inter_sel, (out_inter_sel ? 1 : 0),
    			swap_endianness);
    } else {
    	used_count = apply_datablock_to_buffer_nonlocal_selections(
    			raw_varinfo, transinfo,
    			datablock, output_buffer, output_sel,
    			&inter_sel, (out_inter_sel ? 1 : 0),
    			swap_endianness);
    }

    // Clean up the returned intersection if it is not wanted by the caller
	if (inter_sel) {
		if (out_inter_sel) {
			*out_inter_sel = inter_sel;
		} else {
			// TODO: Deep delete the selection (delete points list, start/count arrays, etc.)
			a2sel_free(inter_sel);
		}
	}

    // Free the datablock
    adios_datablock_free(&datablock, 1);
    return used_count;
}