Exemple #1
0
static unsigned long amd_get_page_size(void *client_context)
{
	unsigned long page_size;
	int result;
	struct amd_mem_context *mem_context =
		(struct amd_mem_context *)client_context;

	MSG_DBG("get_page_size: context: %p\n", client_context);
	MSG_DBG("get_page_size: pid 0x%p, address 0x%llx, size:0x%llx\n",
			mem_context->pid,
			mem_context->va,
			mem_context->size);


	result = rdma_interface->get_page_size(
				mem_context->va,
				mem_context->size,
				mem_context->pid,
				&page_size);

	if (result) {
		MSG_ERR("Could not get page size. %d", result);
		/* If we failed to get page size then do not know what to do.
		 * Let's return some default value
		 */
		return 4096;
	}

	return page_size;
}
Exemple #2
0
static void amd_put_pages(struct sg_table *sg_head, void *client_context)
{
	int ret = 0;
	struct amd_mem_context *mem_context =
		(struct amd_mem_context *)client_context;

	MSG_DBG("put_pages: sg_head %p client_context: 0x%p\n",
			sg_head, client_context);
	MSG_DBG("put_pages: pid 0x%p, address 0x%llx, size:0x%llx\n",
			mem_context->pid,
			mem_context->va,
			mem_context->size);

	MSG_DBG("put_pages: mem_context->p2p_info %p\n",
				mem_context->p2p_info);

	if (ACCESS_ONCE(mem_context->free_callback_called)) {
		MSG_DBG("put_pages: free callback was called\n");
		return;
	}

	if (mem_context->p2p_info) {
		ret = rdma_interface->put_pages(&mem_context->p2p_info);
		mem_context->p2p_info = NULL;

		if (ret)
			MSG_ERR("put_pages failure: %d (callback status %d)",
					ret, mem_context->free_callback_called);
	} else
		MSG_ERR("put_pages: Pointer to p2p info is null\n");
}
Exemple #3
0
static int amd_acquire(unsigned long addr, size_t size,
			void *peer_mem_private_data,
			char *peer_mem_name, void **client_context)
{
	int ret;
	struct amd_mem_context *mem_context;
	struct pid *pid;

	/* Get pointer to structure describing current process */
	pid = get_task_pid(current, PIDTYPE_PID);

	MSG_DBG("acquire: addr:0x%lx,size:0x%x, pid 0x%p\n",
					addr, (unsigned int)size, pid);

	/* Check if it is address handled by AMD GPU driver */
	ret = rdma_interface->is_gpu_address(addr, pid);

	if (!ret) {
		MSG_DBG("acquire: Not GPU Address\n");
		/* This is not GPU address */
		return 0;
	}

	MSG_DBG("acquire: GPU address\n");

	/* Initialize context used for operation with given address */
	mem_context = kzalloc(sizeof(struct amd_mem_context), GFP_KERNEL);

	if (!mem_context) {
		MSG_ERR("failure to allocate memory for mem_context\n");
		/* Error case handled as not GPU address  */
		return 0;
	}

	mem_context->free_callback_called = 0;
	mem_context->va   = addr;
	mem_context->size = size;

	/* Save PI. It is guaranteed that such function will be
	 * called in the correct process context as opposite to others.
	 */
	mem_context->pid  = pid;

	MSG_DBG("acquire: Client context %p\n", mem_context);

	/* Return pointer to allocated context */
	*client_context = mem_context;

	/* Increase counter to prevent module unloading */
	__module_get(THIS_MODULE);

	/* Return 1 to inform that it is address which will be handled
	 * by AMD GPU driver
	 */
	return 1;
}
Exemple #4
0
static int amd_get_pages(unsigned long addr, size_t size, int write, int force,
			  struct sg_table *sg_head,
			  void *client_context, void *core_context)
{
	int ret;
	struct amd_mem_context *mem_context =
		(struct amd_mem_context *)client_context;

	MSG_DBG("get_pages: addr:0x%lx,size:0x%x, core_context:%p\n",
						addr, (unsigned int)size, core_context);

	if (!mem_context) {
		MSG_WARN("get_pages: Invalid client context");
		return -EINVAL;
	}

	MSG_DBG("get_pages: pid :0x%p\n", mem_context->pid);


	if (addr != mem_context->va) {
		MSG_WARN("get_pages: Context address (0x%llx) is not the same",
			mem_context->va);
		return -EINVAL;
	}

	if (size != mem_context->size) {
		MSG_WARN("get_pages: Context size (0x%llx) is not the same",
			mem_context->size);
		return -EINVAL;
	}

	ret = rdma_interface->get_pages(addr,
					size,
					mem_context->pid,
					&mem_context->p2p_info,
					free_callback,
					mem_context);

	if (ret || !mem_context->p2p_info) {
		MSG_ERR("Could not rdma::get_pages failure: %d", ret);
		return ret;
	}

	mem_context->core_context = core_context;

	/* Note: At this stage it is OK not to fill sg_table */
	return 0;
}
Exemple #5
0
static void amd_release(void *client_context)
{
	struct amd_mem_context *mem_context =
		(struct amd_mem_context *)client_context;

	MSG_DBG("release: context: 0x%p\n", client_context);
	MSG_DBG("release: pid 0x%p, address 0x%llx, size:0x%llx\n",
			mem_context->pid,
			mem_context->va,
			mem_context->size);

	kfree(mem_context);

	/* Decrease counter to allow module unloading */
	module_put(THIS_MODULE);
}
Exemple #6
0
static int amd_dma_unmap(struct sg_table *sg_head, void *client_context,
			   struct device  *dma_device)
{
	struct amd_mem_context *mem_context =
		(struct amd_mem_context *)client_context;

	MSG_DBG("dma_unmap: Context 0x%p, sg_table 0x%p\n",
			client_context, sg_head);

	MSG_DBG("dma_unmap: pid 0x%p, address 0x%llx, size:0x%llx\n",
			mem_context->pid,
			mem_context->va,
			mem_context->size);

	/* Assume success */
	return 0;
}
Exemple #7
0
static int amd_dma_map(struct sg_table *sg_head, void *client_context,
			struct device *dma_device, int dmasync, int *nmap)
{
	/*
	 * NOTE/TODO:
	 * We could have potentially three cases for real memory
	 *	location:
	 *		- all memory in the local
	 *		- all memory in the system
	 *		- memory is spread (s/g) between local and system.
	 *
	 *	In the case of all memory in the system we could use
	 *	iommu driver to build DMA addresses but not in the case
	 *	of local memory because currently iommu driver doesn't
	 *	deal with local/device memory addresses (it requires "struct
	 *	page").
	 *
	 *	Accordingly there return is assumption that iommu funcutionality
	 *	should be disabled so we could assume that sg_table already
	 *	contains DMA addresses.
	 *
	 */
	struct amd_mem_context *mem_context =
		(struct amd_mem_context *)client_context;

	MSG_DBG("dma_map: Context 0x%p, sg_head 0x%p\n",
			client_context, sg_head);

	MSG_DBG("dma_map: pid 0x%p, address 0x%llx, size:0x%llx\n",
			mem_context->pid,
			mem_context->va,
			mem_context->size);

	if (!mem_context->p2p_info) {
		MSG_ERR("dma_map: No sg table were allocated\n");
		return -EINVAL;
	}

	/* Copy information about previosly allocate sg_table */
	*sg_head = *mem_context->p2p_info->pages;

	/* Return number of pages */
	*nmap = mem_context->p2p_info->pages->nents;

	return 0;
}
Exemple #8
0
static void free_callback(void *client_priv)
{
	struct amd_mem_context *mem_context =
		(struct amd_mem_context *)client_priv;

	MSG_DBG("free_callback: data 0x%p\n", mem_context);

	if (!mem_context) {
		MSG_WARN("free_callback: Invalid client context");
		return;
	}

	MSG_DBG("mem_context->core_context 0x%p\n", mem_context->core_context);

	/* Call back IB stack asking to invalidate memory */
	(*ib_invalidate_callback) (ib_reg_handle, mem_context->core_context);

	/* amdkfd will freed resources when we returned from this callback.
	 * Set flag to inform that there is nothing to do on "put_pages", etc.
	 */
	ACCESS_ONCE(mem_context->free_callback_called) = 1;
}
Exemple #9
0
MSG_ERR_T msgi_copy_to_sbp(int buf_idx, void *data,int sizeb)
{
    volatile msgi_sbp_meta_t *local_meta_ptr = NULL;
    volatile long local_buffer;
    unsigned char *p;
    int bcount;
    int index;
    int i;
    int level = 3;

    MSG_DBG(level, MESSAGE, "In msgmi_copy_to_sbp()");

    local_meta_ptr = (msgi_sbp_meta_t *) (uintptr_t)msgi_sbp_meta[local_rank][buf_idx];
    local_buffer = msgi_sbp[local_rank][buf_idx];

    MSG_DBG(level, MESSAGE, "(before)producer = %d", local_meta_ptr->producer);

    if(sizeb > block_size)   // now block size = 16KB
    {
        p = (unsigned char *)data;

        //block count
        bcount = msgi_count_blocks(sizeb);
        MSG_DBG(level, MESSAGE, "bcount = %d", bcount);

        //example:  data_size = 120K, block_size = 16K, buffer_size = 64K
        //          bcount = 8, for entry[0-6] is 16K and entry[7] is 120-(8-1)*16K = 8K
        //The for loop is handling the entry[0-6]
        for(i=0;i<(bcount-1);i++)
        {
            index = (local_meta_ptr->producer + 1) % MSGI_MAX_SBP_BLOCKS_PER_BUFFER;

            //when the producer is faster than consumer
            while(index == local_meta_ptr->consumer){}

            MSG_DBG(level, MESSAGE, "addr = %ld", (long)GET_ADDR(local_buffer, local_meta_ptr->producer, block_size));
            memcpy(GET_ADDR(local_buffer,local_meta_ptr->producer,block_size), p, block_size);

            local_meta_ptr->producer = index;

            p += block_size;
        }

        //handle the remain data, e.g. entry[7]
        index = (local_meta_ptr->producer + 1) % MSGI_MAX_SBP_BLOCKS_PER_BUFFER;

        while(index == local_meta_ptr->consumer){}

        MSG_DBG(level, MESSAGE, "addr = %ld", (long)GET_ADDR(local_buffer, local_meta_ptr->producer, block_size));
        memcpy(GET_ADDR(local_buffer,local_meta_ptr->producer,block_size), p,sizeb - (bcount - 1)* block_size);

        local_meta_ptr->producer = index;

        //mark the entry to BUSY
        local_meta_ptr->control_flag = BUSY;
    }
    else
    {
        index = (local_meta_ptr->producer + 1) % MSGI_MAX_SBP_BLOCKS_PER_BUFFER;

        //block is full
        while(index == local_meta_ptr->consumer){}

        MSG_DBG(level, MESSAGE, "addr = %ld", (long)GET_ADDR(local_buffer, local_meta_ptr->producer, block_size));
        memcpy(GET_ADDR(local_buffer,local_meta_ptr->producer,block_size),data,sizeb);

        local_meta_ptr->producer = index;

        //mark the entry to BUSY
        local_meta_ptr->control_flag = BUSY;
    }
    MSG_DBG(level, MESSAGE, "(after)producer = %d", local_meta_ptr->producer);

    MSG_DBG(level, MESSAGE, "Leaving msgmi_copy_to_shm() ...");

    return MSG_SUCCESS;
}
Exemple #10
0
MSG_ERR_T msgi_copy_from_sbp(int src_rank,int buf_idx, void *data,int sizeb)
{
    volatile msgi_sbp_meta_t *remote_meta_ptr = NULL;
    volatile long remote_buffer;
    volatile unsigned char *p;
    volatile int index;
    int bcount;
    int i;
    int level = 3;

    MSG_DBG(level, MESSAGE, "In msgmi_copy_from_sbp()");

    remote_meta_ptr = (msgi_sbp_meta_t *)(uintptr_t)msgi_sbp_meta[src_rank][buf_idx];
    remote_buffer = msgi_sbp[src_rank][buf_idx];

    MSG_DBG(level, MESSAGE, "(before)consumer = %d", remote_meta_ptr->consumer);

    if(sizeb > block_size)   // now block size = 16KB
    {
        p = (unsigned char *)data;

        bcount = msgi_count_blocks(sizeb);
        MSG_DBG(level, MESSAGE, "bcount = %d", bcount);

        for(i=0;i<(bcount-1);i++)
        {
            // Check if the shared memory queue is not empty
            while(remote_meta_ptr->consumer == remote_meta_ptr->producer){}

            memcpy((void *)p, GET_ADDR(remote_buffer,remote_meta_ptr->consumer,block_size), block_size);

            index = (remote_meta_ptr->consumer + 1) % MSGI_MAX_SBP_BLOCKS_PER_BUFFER;
            remote_meta_ptr->consumer = index;

            p += block_size;
        }

        // Check if the shared memory queue is not empty
        while(remote_meta_ptr->consumer == remote_meta_ptr->producer){}

        memcpy((void *)p, GET_ADDR(remote_buffer,remote_meta_ptr->consumer,block_size), sizeb - (bcount - 1) * block_size);

        index = (remote_meta_ptr->consumer + 1) % MSGI_MAX_SBP_BLOCKS_PER_BUFFER;
        remote_meta_ptr->consumer = index;

        //mark the entry to FREE
        remote_meta_ptr->control_flag = FREE;
    }
    else
    {
        //Check if the shared memory queue is not empty
        while(remote_meta_ptr->consumer == remote_meta_ptr->producer){}

        memcpy(data,GET_ADDR(remote_buffer,remote_meta_ptr->consumer,block_size),sizeb);

        index = (remote_meta_ptr->consumer + 1) % MSGI_MAX_SBP_BLOCKS_PER_BUFFER;

        remote_meta_ptr->consumer = index;

        //mark the entry to FREE
        remote_meta_ptr->control_flag = FREE;
    }

    MSG_DBG(level, MESSAGE, "(after)consumer = %d", remote_meta_ptr->consumer);
    MSG_DBG(level, MESSAGE, "Leaving msgmi_copy_from_sbp() ...");

    return MSG_SUCCESS;
}