projects / xeon-phi-kernel-module / blob
? search:


870746c2e16d0b65947063c1186ba0a1171655ab
[xeon-phi-kernel-module] / micscif / micscif_rma.c
/*
* Copyright 2010-2017 Intel Corporation.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License, version 2,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* Disclaimer: The codes contained in these modules may be specific to
* the Intel Software Development Platform codenamed Knights Ferry,
* and the Intel product codenamed Knights Corner, and are not backward
* compatible with other Intel products. Additionally, Intel will NOT
* support the codes or instruction set in future products.
*
* Intel offers no warranty of any kind regarding the code. This code is
* licensed on an "AS IS" basis and Intel is not obligated to provide
* any support, assistance, installation, training, or other services
* of any kind. Intel is also not obligated to provide any updates,
* enhancements or extensions. Intel specifically disclaims any warranty
* of merchantability, non-infringement, fitness for any particular
* purpose, and any other warranty.
*
* Further, Intel disclaims all liability of any kind, including but
* not limited to liability for infringement of any proprietary rights,
* relating to the use of the code, even if Intel is notified of the
* possibility of such liability. Except as expressly stated in an Intel
* license agreement provided with this code and agreed upon with Intel,
* no license, express or implied, by estoppel or otherwise, to any
* intellectual property rights is granted herein.
*/
#include "mic/micscif.h"
#include "mic/micscif_smpt.h"
#include "mic/micscif_kmem_cache.h"
#include "mic/micscif_rma_list.h"
#ifndef _MIC_SCIF_
#include "mic_common.h"
#endif
#include "mic/mic_dma_api.h"
#include "mic/micscif_map.h"
bool mic_reg_cache_enable = 0;
bool mic_huge_page_enable = 1;
#ifdef _MIC_SCIF_
mic_dma_handle_t mic_dma_handle;
#endif
static inline
void micscif_rma_destroy_tcw(struct rma_mmu_notifier *mmn,
struct endpt *ep, bool inrange,
uint64_t start, uint64_t len);
#ifdef CONFIG_MMU_NOTIFIER
static void scif_mmu_notifier_release(struct mmu_notifier *mn,
struct mm_struct *mm);
static void scif_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
struct mm_struct *mm,
unsigned long address);
static void scif_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
struct mm_struct *mm,
unsigned long start, unsigned long end);
static void scif_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
struct mm_struct *mm,
unsigned long start, unsigned long end);
static const struct mmu_notifier_ops scif_mmu_notifier_ops = {
.release = scif_mmu_notifier_release,
.clear_flush_young = NULL,
.change_pte = NULL,/*TODO*/
.invalidate_page = scif_mmu_notifier_invalidate_page,
.invalidate_range_start = scif_mmu_notifier_invalidate_range_start,
.invalidate_range_end = scif_mmu_notifier_invalidate_range_end};
static void scif_mmu_notifier_release(struct mmu_notifier *mn,
struct mm_struct *mm)
{
struct endpt *ep;
struct rma_mmu_notifier *mmn;
mmn = container_of(mn, struct rma_mmu_notifier, ep_mmu_notifier);
ep = mmn->ep;
micscif_rma_destroy_tcw(mmn, ep, false, 0, 0);
pr_debug("%s\n", __func__);
return;
}
static void scif_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
struct mm_struct *mm,
unsigned long address)
{
struct endpt *ep;
struct rma_mmu_notifier *mmn;
mmn = container_of(mn, struct rma_mmu_notifier, ep_mmu_notifier);
ep = mmn->ep;
micscif_rma_destroy_tcw(mmn, ep, true, address, PAGE_SIZE);
pr_debug("%s address 0x%lx\n", __func__, address);
return;
}
static void scif_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
struct mm_struct *mm,
unsigned long start, unsigned long end)
{
struct endpt *ep;
struct rma_mmu_notifier *mmn;
mmn = container_of(mn, struct rma_mmu_notifier, ep_mmu_notifier);
ep = mmn->ep;
micscif_rma_destroy_tcw(mmn, ep, true, (uint64_t)start, (uint64_t)(end - start));
pr_debug("%s start=%lx, end=%lx\n", __func__, start, end);
return;
}
static void scif_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
struct mm_struct *mm,
unsigned long start, unsigned long end)
{
/* Nothing to do here, everything needed was done in invalidate_range_start */
pr_debug("%s\n", __func__);
return;
}
#endif
#ifdef CONFIG_MMU_NOTIFIER
void ep_unregister_mmu_notifier(struct endpt *ep)
{
struct endpt_rma_info *rma = &ep->rma_info;
struct rma_mmu_notifier *mmn = NULL;
struct list_head *item, *tmp;
mutex_lock(&ep->rma_info.mmn_lock);
list_for_each_safe(item, tmp, &rma->mmn_list) {
mmn = list_entry(item,
struct rma_mmu_notifier, list_member);
mmu_notifier_unregister(&mmn->ep_mmu_notifier, mmn->mm);
#ifdef RMA_DEBUG
BUG_ON(atomic_long_sub_return(1, &ms_info.mmu_notif_cnt) < 0);
#endif
list_del(item);
kfree(mmn);
}
mutex_unlock(&ep->rma_info.mmn_lock);
}
static void init_mmu_notifier(struct rma_mmu_notifier *mmn, struct mm_struct *mm, struct endpt *ep)
{
mmn->ep = ep;
mmn->mm = mm;
mmn->ep_mmu_notifier.ops = &scif_mmu_notifier_ops;
INIT_LIST_HEAD(&mmn->list_member);
INIT_LIST_HEAD(&mmn->tc_reg_list);
}
static struct rma_mmu_notifier *find_mmu_notifier(struct mm_struct *mm, struct endpt_rma_info *rma)
{
struct rma_mmu_notifier *mmn;
struct list_head *item;
list_for_each(item, &rma->mmn_list) {
mmn = list_entry(item,
struct rma_mmu_notifier, list_member);
if (mmn->mm == mm)
return mmn;
}
return NULL;
}
#endif
/**
* micscif_rma_ep_init:
* @ep: end point
*
* Initialize RMA per EP data structures.
*/
int micscif_rma_ep_init(struct endpt *ep)
{
int ret;
struct endpt_rma_info *rma = &ep->rma_info;
mutex_init (&rma->rma_lock);
if ((ret = va_gen_init(&rma->va_gen,
VA_GEN_MIN, VA_GEN_RANGE)) < 0)
goto init_err;
spin_lock_init(&rma->tc_lock);
mutex_init (&rma->mmn_lock);
mutex_init (&rma->va_lock);
INIT_LIST_HEAD(&rma->reg_list);
INIT_LIST_HEAD(&rma->remote_reg_list);
atomic_set(&rma->tw_refcount, 0);
atomic_set(&rma->tw_total_pages, 0);
atomic_set(&rma->tcw_refcount, 0);
atomic_set(&rma->tcw_total_pages, 0);
init_waitqueue_head(&rma->fence_wq);
rma->fence_refcount = 0;
rma->async_list_del = 0;
rma->dma_chan = NULL;
INIT_LIST_HEAD(&rma->mmn_list);
INIT_LIST_HEAD(&rma->task_list);
init_err:
return ret;
}
/**
* micscif_rma_ep_can_uninit:
* @ep: end point
*
* Returns 1 if an endpoint can be uninitialized and 0 otherwise.
*/
int micscif_rma_ep_can_uninit(struct endpt *ep)
{
int ret = 0;
/* Destroy RMA Info only if both lists are empty */
if (list_empty(&ep->rma_info.reg_list) &&
list_empty(&ep->rma_info.remote_reg_list) &&
#ifdef CONFIG_MMU_NOTIFIER
list_empty(&ep->rma_info.mmn_list) &&
#endif
!atomic_read(&ep->rma_info.tw_refcount) &&
!atomic_read(&ep->rma_info.tcw_refcount))
ret = 1;
return ret;
}
#ifdef _MIC_SCIF_
/**
* __micscif_setup_proxy_dma:
* @ep: SCIF endpoint descriptor.
*
* Sets up data structures for P2P Proxy DMAs.
*/
static int __micscif_setup_proxy_dma(struct endpt *ep)
{
struct endpt_rma_info *rma = &ep->rma_info;
int err = 0;
uint64_t *tmp = NULL;
mutex_lock(&rma->rma_lock);
if (is_p2p_scifdev(ep->remote_dev) && !rma->proxy_dma_va) {
if (!(tmp = scif_zalloc(PAGE_SIZE))) {
err = -ENOMEM;
goto error;
}
if ((err = map_virt_into_aperture(&rma->proxy_dma_phys,
tmp,
ep->remote_dev, PAGE_SIZE))) {
scif_free(tmp, PAGE_SIZE);
goto error;
}
*tmp = OP_IDLE;
rma->proxy_dma_va = tmp;
}
error:
mutex_unlock(&rma->rma_lock);
return err;
}
static __always_inline int micscif_setup_proxy_dma(struct endpt *ep)
{
if (ep->rma_info.proxy_dma_va)
return 0;
return __micscif_setup_proxy_dma(ep);
}
/**
* micscif_teardown_proxy_dma:
* @ep: SCIF endpoint descriptor.
*
* Tears down data structures setup for P2P Proxy DMAs.
*/
void micscif_teardown_proxy_dma(struct endpt *ep)
{
struct endpt_rma_info *rma = &ep->rma_info;
mutex_lock(&rma->rma_lock);
if (rma->proxy_dma_va) {
unmap_from_aperture(rma->proxy_dma_phys, ep->remote_dev, PAGE_SIZE);
scif_free(rma->proxy_dma_va, PAGE_SIZE);
rma->proxy_dma_va = NULL;
}
mutex_unlock(&rma->rma_lock);
}
/**
* micscif_proxy_dma:
* @ep: SCIF endpoint descriptor.
* @copy_work: DMA copy work information.
*
* This API does the following:
* 1) Sends the peer a SCIF Node QP message with the information
* required to program a proxy DMA to covert a P2P Read to a Write
* which will initiate a DMA transfer from the peer card to self.
* The reason for this special code path is KNF and KNC P2P read
* performance being much lower than P2P write performance on Crown
* Pass platforms.
* 2) Poll for an update of the known proxy dma VA to OP_COMPLETED
* via a SUD by the peer.
*/
static int micscif_proxy_dma(scif_epd_t epd, struct mic_copy_work *work)
{
struct endpt *ep = (struct endpt *)epd;
struct nodemsg msg;
unsigned long ts = jiffies;
struct endpt_rma_info *rma = &ep->rma_info;
int err;
volatile uint64_t *proxy_dma_va = rma->proxy_dma_va;
mutex_lock(&ep->rma_info.rma_lock);
/*
* Bail out if there is a Proxy DMA already in progress
* for this endpoint. The callee will fallback on self
* DMAs upon an error.
*/
if (*proxy_dma_va != OP_IDLE) {
mutex_unlock(&ep->rma_info.rma_lock);
err = -EBUSY;
goto error;
}
*proxy_dma_va = OP_IN_PROGRESS;
mutex_unlock(&ep->rma_info.rma_lock);
msg.src = ep->port;
msg.uop = work->ordered ? SCIF_PROXY_ORDERED_DMA : SCIF_PROXY_DMA;
msg.payload[0] = ep->remote_ep;
msg.payload[1] = work->src_offset;
msg.payload[2] = work->dst_offset;
msg.payload[3] = work->len;
if ((err = micscif_nodeqp_send(ep->remote_dev, &msg, ep)))
goto error_init_va;
while (*proxy_dma_va != OP_COMPLETED) {
schedule();
if (time_after(jiffies,
ts + NODE_ALIVE_TIMEOUT)) {
err = -EBUSY;
goto error_init_va;
}
}
err = 0;
error_init_va:
*proxy_dma_va = OP_IDLE;
error:
return err;
}
#endif
/**
* micscif_create_pinned_pages:
* @nr_pages: number of pages in window
* @prot: read/write protection
*
* Allocate and prepare a set of pinned pages.
*/
struct scif_pinned_pages *micscif_create_pinned_pages(int nr_pages, int prot)
{
struct scif_pinned_pages *pinned_pages;
might_sleep();
if (!(pinned_pages = scif_zalloc(sizeof(*pinned_pages))))
goto error;
if (!(pinned_pages->pages = scif_zalloc(nr_pages *
sizeof(*(pinned_pages->pages)))))
goto error_free_pinned_pages;
if (!(pinned_pages->num_pages = scif_zalloc(nr_pages *
sizeof(*(pinned_pages->num_pages)))))
goto error_free_pages;
#if !defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_HUGETLB_PAGE) && !defined(_MIC_SCIF_)
if (!(pinned_pages->vma = scif_zalloc(nr_pages *
sizeof(*(pinned_pages->vma)))))
goto error_free_num_pages;
#endif
pinned_pages->prot = prot;
pinned_pages->magic = SCIFEP_MAGIC;
pinned_pages->nr_contig_chunks = 0;
return pinned_pages;
#if !defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_HUGETLB_PAGE) && !defined(_MIC_SCIF_)
error_free_num_pages:
scif_free(pinned_pages->num_pages,
pinned_pages->nr_pages * sizeof(*(pinned_pages->num_pages)));
#endif
error_free_pages:
scif_free(pinned_pages->pages,
pinned_pages->nr_pages * sizeof(*(pinned_pages->pages)));
error_free_pinned_pages:
scif_free(pinned_pages, sizeof(*pinned_pages));
error:
return NULL;
}
/**
* micscif_destroy_pinned_pages:
* @pinned_pages: A set of pinned pages.
*
* Deallocate resources for pinned pages.
*/
int micscif_destroy_pinned_pages(struct scif_pinned_pages *pinned_pages)
{
int j;
int writeable = pinned_pages->prot & SCIF_PROT_WRITE;
int kernel = SCIF_MAP_KERNEL & pinned_pages->map_flags;
for (j = 0; j < pinned_pages->nr_pages; j++) {
if (pinned_pages->pages[j]) {
if (!kernel) {
if (writeable)
SetPageDirty(pinned_pages->pages[j]);
#ifdef RMA_DEBUG
BUG_ON(!page_count(pinned_pages->pages[j]));
BUG_ON(atomic_long_sub_return(1, &ms_info.rma_pin_cnt) < 0);
#endif
put_page(pinned_pages->pages[j]);
}
}
}
#if !defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_HUGETLB_PAGE) && !defined(_MIC_SCIF_)
scif_free(pinned_pages->vma,
pinned_pages->nr_pages * sizeof(*(pinned_pages->vma)));
#endif
scif_free(pinned_pages->pages,
pinned_pages->nr_pages * sizeof(*(pinned_pages->pages)));
scif_free(pinned_pages->num_pages,
pinned_pages->nr_pages * sizeof(*(pinned_pages->num_pages)));
scif_free(pinned_pages, sizeof(*pinned_pages));
return 0;
}
/*
* micscif_create_window:
* @ep: end point
* @pinned_pages: Set of pinned pages which wil back this window.
* @offset: offset hint
*
* Allocate and prepare a self registration window.
*/
struct reg_range_t *micscif_create_window(struct endpt *ep,
int64_t nr_pages, uint64_t offset, bool temp)
{
struct reg_range_t *window;
might_sleep();
if (!(window = scif_zalloc(sizeof(struct reg_range_t))))
goto error;
#ifdef CONFIG_ML1OM
if (!temp) {
if (!(window->phys_addr = scif_zalloc(nr_pages *
sizeof(*(window->phys_addr)))))
goto error_free_window;
if (!(window->temp_phys_addr = scif_zalloc(nr_pages *
sizeof(*(window->temp_phys_addr)))))
goto error_free_window;
}
#endif
if (!(window->dma_addr = scif_zalloc(nr_pages *
sizeof(*(window->dma_addr)))))
goto error_free_window;
if (!(window->num_pages = scif_zalloc(nr_pages *
sizeof(*(window->num_pages)))))
goto error_free_window;
window->offset = offset;
window->ep = (uint64_t)ep;
window->magic = SCIFEP_MAGIC;
window->reg_state = OP_IDLE;
init_waitqueue_head(&window->regwq);
window->unreg_state = OP_IDLE;
init_waitqueue_head(&window->unregwq);
INIT_LIST_HEAD(&window->list_member);
window->type = RMA_WINDOW_SELF;
window->temp = temp;
#ifdef _MIC_SCIF_
micscif_setup_proxy_dma(ep);
#endif
return window;
error_free_window:
if (window->dma_addr)
scif_free(window->dma_addr, nr_pages * sizeof(*(window->dma_addr)));
#ifdef CONFIG_ML1OM
if (window->temp_phys_addr)
scif_free(window->temp_phys_addr, nr_pages * sizeof(*(window->temp_phys_addr)));
if (window->phys_addr)
scif_free(window->phys_addr, nr_pages * sizeof(*(window->phys_addr)));
#endif
scif_free(window, sizeof(*window));
error:
return NULL;
}
/**
* micscif_destroy_incomplete_window:
* @ep: end point
* @window: registration window
*
* Deallocate resources for self window.
*/
int micscif_destroy_incomplete_window(struct endpt *ep, struct reg_range_t *window)
{
int err;
int64_t nr_pages = window->nr_pages;
struct allocmsg *alloc = &window->alloc_handle;
struct nodemsg msg;
RMA_MAGIC(window);
retry:
err = wait_event_timeout(alloc->allocwq, alloc->state != OP_IN_PROGRESS, NODE_ALIVE_TIMEOUT);
if (!err && scifdev_alive(ep))
goto retry;
if (OP_COMPLETED == alloc->state) {
msg.uop = SCIF_FREE_VIRT;
msg.src = ep->port;
msg.payload[0] = ep->remote_ep;
msg.payload[1] = (uint64_t)window->alloc_handle.vaddr;
msg.payload[2] = (uint64_t)window;
msg.payload[3] = SCIF_REGISTER;
micscif_nodeqp_send(ep->remote_dev, &msg, ep);
}
micscif_free_window_offset(ep, window->offset,
window->nr_pages << PAGE_SHIFT);
if (window->dma_addr)
scif_free(window->dma_addr, nr_pages *
sizeof(*(window->dma_addr)));
if (window->num_pages)
scif_free(window->num_pages, nr_pages *
sizeof(*(window->num_pages)));
#ifdef CONFIG_ML1OM
if (window->phys_addr)
scif_free(window->phys_addr, window->nr_pages *
sizeof(*(window->phys_addr)));
if (window->temp_phys_addr)
scif_free(window->temp_phys_addr, nr_pages *
sizeof(*(window->temp_phys_addr)));
#endif
scif_free(window, sizeof(*window));
return 0;
}
/**
* micscif_destroy_window:
* @ep: end point
* @window: registration window
*
* Deallocate resources for self window.
*/
int micscif_destroy_window(struct endpt *ep, struct reg_range_t *window)
{
int j;
struct scif_pinned_pages *pinned_pages = window->pinned_pages;
int64_t nr_pages = window->nr_pages;
might_sleep();
RMA_MAGIC(window);
if (!window->temp && window->mm) {
__scif_dec_pinned_vm_lock(window->mm, window->nr_pages, 0);
__scif_release_mm(window->mm);
window->mm = NULL;
}
if (!window->offset_freed)
micscif_free_window_offset(ep, window->offset,
window->nr_pages << PAGE_SHIFT);
for (j = 0; j < window->nr_contig_chunks; j++) {
if (window->dma_addr[j]) {
unmap_from_aperture(
window->dma_addr[j],
ep->remote_dev,
window->num_pages[j] << PAGE_SHIFT);
}
}
/*
* Decrement references for this set of pinned pages from
* this window.
*/
j = atomic_sub_return((int32_t)pinned_pages->nr_pages,
&pinned_pages->ref_count);
BUG_ON(j < 0);
/*
* If the ref count for pinned_pages is zero then someone
* has already called scif_unpin_pages() for it and we should
* destroy the page cache.
*/
if (!j)
micscif_destroy_pinned_pages(window->pinned_pages);
if (window->dma_addr)
scif_free(window->dma_addr, nr_pages *
sizeof(*(window->dma_addr)));
if (window->num_pages)
scif_free(window->num_pages, nr_pages *
sizeof(*(window->num_pages)));
#ifdef CONFIG_ML1OM
if (window->phys_addr)
scif_free(window->phys_addr, window->nr_pages *
sizeof(*(window->phys_addr)));
if (window->temp_phys_addr)
scif_free(window->temp_phys_addr, nr_pages *
sizeof(*(window->temp_phys_addr)));
#endif
window->magic = 0;
scif_free(window, sizeof(*window));
return 0;
}
/**
* micscif_create_remote_lookup:
* @ep: end point
* @window: remote window
*
* Allocate and prepare lookup entries for the remote
* end to copy over the physical addresses.
* Returns 0 on success and appropriate errno on failure.
*/
int micscif_create_remote_lookup(struct endpt *ep, struct reg_range_t *window)
{
int i, j, err = 0;
int64_t nr_pages = window->nr_pages;
bool vmalloc_dma_phys;
#ifdef CONFIG_ML1OM
bool vmalloc_temp_phys = false;
bool vmalloc_phys = false;
#endif
might_sleep();
/* Map window */
err = map_virt_into_aperture(&window->mapped_offset,
window, ep->remote_dev, sizeof(*window));
if (err)
goto error_window;
/* Compute the number of lookup entries. 21 == 2MB Shift */
window->nr_lookup = ALIGN(nr_pages * PAGE_SIZE,
((2) * 1024 * 1024)) >> 21;
if (!(window->dma_addr_lookup.lookup =
scif_zalloc(window->nr_lookup *
sizeof(*(window->dma_addr_lookup.lookup)))))
goto error_window;
/* Map DMA physical addess lookup array */
err = map_virt_into_aperture(&window->dma_addr_lookup.offset,
window->dma_addr_lookup.lookup, ep->remote_dev,
window->nr_lookup *
sizeof(*window->dma_addr_lookup.lookup));
if (err)
goto error_window;
vmalloc_dma_phys = is_vmalloc_addr(&window->dma_addr[0]);
#ifdef CONFIG_ML1OM
if (ep->remote_dev != &scif_dev[SCIF_HOST_NODE] && !is_self_scifdev(ep->remote_dev)) {
if (!(window->temp_phys_addr_lookup.lookup =
scif_zalloc(window->nr_lookup *
sizeof(*(window->temp_phys_addr_lookup.lookup)))))
goto error_window;
/* Map physical addess lookup array */
err = map_virt_into_aperture(&window->temp_phys_addr_lookup.offset,
window->temp_phys_addr_lookup.lookup, ep->remote_dev,
window->nr_lookup *
sizeof(*window->temp_phys_addr_lookup.lookup));
if (err)
goto error_window;
if (!(window->phys_addr_lookup.lookup =
scif_zalloc(window->nr_lookup *
sizeof(*(window->phys_addr_lookup.lookup)))))
goto error_window;
/* Map physical addess lookup array */
err = map_virt_into_aperture(&window->phys_addr_lookup.offset,
window->phys_addr_lookup.lookup, ep->remote_dev,
window->nr_lookup *
sizeof(*window->phys_addr_lookup.lookup));
if (err)
goto error_window;
vmalloc_phys = is_vmalloc_addr(&window->phys_addr[0]);
vmalloc_temp_phys = is_vmalloc_addr(&window->temp_phys_addr[0]);
}
#endif
/* Now map each of the pages containing physical addresses */
for (i = 0, j = 0; i < nr_pages; i += NR_PHYS_ADDR_IN_PAGE, j++) {
#ifdef CONFIG_ML1OM
if (ep->remote_dev != &scif_dev[SCIF_HOST_NODE] && !is_self_scifdev(ep->remote_dev)) {
err = map_page_into_aperture(
&window->temp_phys_addr_lookup.lookup[j],
vmalloc_temp_phys ?
vmalloc_to_page(&window->temp_phys_addr[i]) :
virt_to_page(&window->temp_phys_addr[i]),
ep->remote_dev);
if (err)
goto error_window;
err = map_page_into_aperture(
&window->phys_addr_lookup.lookup[j],
vmalloc_phys ?
vmalloc_to_page(&window->phys_addr[i]) :
virt_to_page(&window->phys_addr[i]),
ep->remote_dev);
if (err)
goto error_window;
}
#endif
err = map_page_into_aperture(
&window->dma_addr_lookup.lookup[j],
vmalloc_dma_phys ?
vmalloc_to_page(&window->dma_addr[i]) :
virt_to_page(&window->dma_addr[i]),
ep->remote_dev);
if (err)
goto error_window;
}
return 0;
error_window:
return err;
}
/**
* micscif_destroy_remote_lookup:
* @ep: end point
* @window: remote window
*
* Destroy lookup entries used for the remote
* end to copy over the physical addresses.
*/
void micscif_destroy_remote_lookup(struct endpt *ep, struct reg_range_t *window)
{
int i, j;
RMA_MAGIC(window);
if (window->nr_lookup) {
for (i = 0, j = 0; i < window->nr_pages;
i += NR_PHYS_ADDR_IN_PAGE, j++) {
if (window->dma_addr_lookup.lookup &&
window->dma_addr_lookup.lookup[j]) {
unmap_from_aperture(
window->dma_addr_lookup.lookup[j],
ep->remote_dev, PAGE_SIZE);
}
}
if (window->dma_addr_lookup.offset) {
unmap_from_aperture(
window->dma_addr_lookup.offset,
ep->remote_dev, window->nr_lookup *
sizeof(*window->dma_addr_lookup.lookup));
}
if (window->dma_addr_lookup.lookup)
scif_free(window->dma_addr_lookup.lookup, window->nr_lookup *
sizeof(*(window->dma_addr_lookup.lookup)));
if (window->mapped_offset) {
unmap_from_aperture(window->mapped_offset,
ep->remote_dev, sizeof(*window));
}
window->nr_lookup = 0;
}
}
/**
* micscif_create_remote_window:
* @ep: end point
* @nr_pages: number of pages in window
*
* Allocate and prepare a remote registration window.
*/
struct reg_range_t *micscif_create_remote_window(struct endpt *ep, int nr_pages)
{
struct reg_range_t *window;
might_sleep();
if (!(window = scif_zalloc(sizeof(struct reg_range_t))))
goto error_ret;
window->magic = SCIFEP_MAGIC;
window->nr_pages = nr_pages;
#if !defined(_MIC_SCIF_) && defined(CONFIG_ML1OM)
if (!(window->page_ref_count = scif_zalloc(nr_pages *
sizeof(*(window->page_ref_count)))))
goto error_window;
#endif
if (!(window->dma_addr = scif_zalloc(nr_pages *
sizeof(*(window->dma_addr)))))
goto error_window;
if (!(window->num_pages = scif_zalloc(nr_pages *
sizeof(*(window->num_pages)))))
goto error_window;
#ifdef CONFIG_ML1OM
if (!(window->phys_addr = scif_zalloc(nr_pages *
sizeof(*(window->phys_addr)))))
goto error_window;
if (!(window->temp_phys_addr = scif_zalloc(nr_pages *
sizeof(*(window->temp_phys_addr)))))
goto error_window;
#endif
if (micscif_create_remote_lookup(ep, window))
goto error_window;
window->ep = (uint64_t)ep;
window->type = RMA_WINDOW_PEER;
set_window_ref_count(window, nr_pages);
window->get_put_ref_count = 0;
window->unreg_state = OP_IDLE;
#if !defined(_MIC_SCIF_) && defined(CONFIG_ML1OM)
window->gttmap_state = OP_IDLE;
init_waitqueue_head(&window->gttmapwq);
#endif
#ifdef _MIC_SCIF_
micscif_setup_proxy_dma(ep);
window->proxy_dma_phys = ep->rma_info.proxy_dma_phys;
#endif
return window;
error_window:
micscif_destroy_remote_window(ep, window);
error_ret:
return NULL;
}
/**
* micscif_destroy_remote_window:
* @ep: end point
* @window: remote registration window
*
* Deallocate resources for remote window.
*/
void micscif_destroy_remote_window(struct endpt *ep, struct reg_range_t *window)
{
RMA_MAGIC(window);
micscif_destroy_remote_lookup(ep, window);
if (window->dma_addr)
scif_free(window->dma_addr, window->nr_pages *
sizeof(*(window->dma_addr)));
if (window->num_pages)
scif_free(window->num_pages, window->nr_pages *
sizeof(*(window->num_pages)));
#ifdef CONFIG_ML1OM
if (window->phys_addr)
scif_free(window->phys_addr, window->nr_pages *
sizeof(*(window->phys_addr)));
if (window->temp_phys_addr)
scif_free(window->temp_phys_addr, window->nr_pages *
sizeof(*(window->temp_phys_addr)));
#endif
#if !defined(_MIC_SCIF_) && defined(CONFIG_ML1OM)
if (window->page_ref_count)
scif_free(window->page_ref_count, window->nr_pages *
sizeof(*(window->page_ref_count)));
#endif
window->magic = 0;
scif_free(window, sizeof(*window));
}
/**
* micscif_map_window_pages:
* @ep: end point
* @window: self registration window
* @tmp_wnd: is a temporary window?
*
* Map pages of a window into the aperture/PCI.
* Also compute physical addresses required for DMA.
*/
int micscif_map_window_pages(struct endpt *ep, struct reg_range_t *window, bool tmp_wnd)
{
int j, i, err = 0, nr_pages;
scif_pinned_pages_t pinned_pages;
might_sleep();
RMA_MAGIC(window);
pinned_pages = window->pinned_pages;
for (j = 0, i = 0; j < window->nr_contig_chunks; j++, i += nr_pages) {
nr_pages = pinned_pages->num_pages[i];
#ifdef _MIC_SCIF_
#ifdef CONFIG_ML1OM
/* phys_addr[] holds addresses as seen from the remote node
* these addressed are then copied into the remote card's
* window structure
* when the remote node is the host and the card is knf
* these addresses are only created at the point of mapping
* the card physical address into gtt (for the KNC the
* the gtt code path returns the local address)
* when the remote node is loopback - the address remains
* the same
* when the remote node is a kn* - the base address of the local
* card as seen from the remote node is added in
*/
if (!tmp_wnd) {
if(ep->remote_dev != &scif_dev[SCIF_HOST_NODE]) {
if ((err = map_virt_into_aperture(
&window->temp_phys_addr[j],
phys_to_virt(page_to_phys(pinned_pages->pages[i])),
ep->remote_dev,
nr_pages << PAGE_SHIFT))) {
int k,l;
for (l = k = 0; k < i; l++) {
nr_pages = pinned_pages->num_pages[k];
window->temp_phys_addr[l]
&= ~RMA_HUGE_NR_PAGE_MASK;
unmap_from_aperture(
window->temp_phys_addr[l],
ep->remote_dev,
nr_pages << PAGE_SHIFT);
k += nr_pages;
window->temp_phys_addr[l] = 0;
}
return err;
}
if (!tmp_wnd)
RMA_SET_NR_PAGES(window->temp_phys_addr[j], nr_pages);
}
}
#endif
window->dma_addr[j] =
page_to_phys(pinned_pages->pages[i]);
if (!tmp_wnd)
RMA_SET_NR_PAGES(window->dma_addr[j], nr_pages);
#else
err = map_virt_into_aperture(&window->dma_addr[j],
phys_to_virt(page_to_phys(pinned_pages->pages[i])),
ep->remote_dev, nr_pages << PAGE_SHIFT);
if (err)
return err;
if (!tmp_wnd)
RMA_SET_NR_PAGES(window->dma_addr[j], nr_pages);
#endif
window->num_pages[j] = nr_pages;
}
return err;
}
/**
* micscif_unregister_window:
* @window: self registration window
*
* Send an unregistration request and wait for a response.
*/
int micscif_unregister_window(struct reg_range_t *window)
{
int err = 0;
struct endpt *ep = (struct endpt *)window->ep;
bool send_msg = false;
might_sleep();
BUG_ON(!mutex_is_locked(&ep->rma_info.rma_lock));
switch (window->unreg_state) {
case OP_IDLE:
{
window->unreg_state = OP_IN_PROGRESS;
send_msg = true;
/* fall through */
}
case OP_IN_PROGRESS:
{
get_window_ref_count(window, 1);
mutex_unlock(&ep->rma_info.rma_lock);
if (send_msg && (err = micscif_send_scif_unregister(ep, window))) {
window->unreg_state = OP_COMPLETED;
goto done;
}
retry:
err = wait_event_timeout(window->unregwq,
window->unreg_state != OP_IN_PROGRESS, NODE_ALIVE_TIMEOUT);
if (!err && scifdev_alive(ep))
goto retry;
if (!err) {
err = -ENODEV;
window->unreg_state = OP_COMPLETED;
printk(KERN_ERR "%s %d err %d\n", __func__, __LINE__, err);
}
if (err > 0)
err = 0;
done:
mutex_lock(&ep->rma_info.rma_lock);
put_window_ref_count(window, 1);
break;
}
case OP_FAILED:
{
if (!scifdev_alive(ep)) {
err = -ENODEV;
window->unreg_state = OP_COMPLETED;
}
break;
}
case OP_COMPLETED:
break;
default:
/* Invalid opcode? */
BUG_ON(1);
}
if (OP_COMPLETED == window->unreg_state &&
window->ref_count)
put_window_ref_count(window, window->nr_pages);
if (!window->ref_count) {
atomic_inc(&ep->rma_info.tw_refcount);
atomic_add_return((int32_t)window->nr_pages, &ep->rma_info.tw_total_pages);
list_del(&window->list_member);
micscif_free_window_offset(ep, window->offset,
window->nr_pages << PAGE_SHIFT);
window->offset_freed = true;
mutex_unlock(&ep->rma_info.rma_lock);
if ((!!(window->pinned_pages->map_flags & SCIF_MAP_KERNEL))
&& scifdev_alive(ep)) {
drain_dma_intr(ep->rma_info.dma_chan);
} else {
if (!__scif_dec_pinned_vm_lock(window->mm,
window->nr_pages, 1)) {
__scif_release_mm(window->mm);
window->mm = NULL;
}
}
micscif_queue_for_cleanup(window, &ms_info.mi_rma);
mutex_lock(&ep->rma_info.rma_lock);
}
return err;
}
/**
* micscif_send_alloc_request:
* @ep: end point
* @window: self registration window
*
* Send a remote window allocation request
*/
int micscif_send_alloc_request(struct endpt *ep, struct reg_range_t *window)
{
struct nodemsg msg;
struct allocmsg *alloc = &window->alloc_handle;
/* Set up the Alloc Handle */
alloc->uop = SCIF_REGISTER;
alloc->state = OP_IN_PROGRESS;
init_waitqueue_head(&alloc->allocwq);
/* Send out an allocation request */
msg.uop = SCIF_ALLOC_REQ;
msg.src = ep->port;
msg.payload[0] = ep->remote_ep;
msg.payload[1] = window->nr_pages;
msg.payload[2] = (uint64_t)&window->alloc_handle;
msg.payload[3] = SCIF_REGISTER;
return micscif_nodeqp_send(ep->remote_dev, &msg, ep);
}
/**
* micscif_prep_remote_window:
* @ep: end point
* @window: self registration window
*
* Send a remote window allocation request, wait for an allocation response,
* prepare the remote window and notify the peer to unmap it once done.
*/
int micscif_prep_remote_window(struct endpt *ep, struct reg_range_t *window)
{
struct nodemsg msg;
struct reg_range_t *remote_window;
struct allocmsg *alloc = &window->alloc_handle;
dma_addr_t *dma_phys_lookup, *tmp;
int i = 0, j = 0;
int nr_contig_chunks, loop_nr_contig_chunks, remaining_nr_contig_chunks, nr_lookup;
#if defined(_MIC_SCIF_) && defined(CONFIG_ML1OM)
dma_addr_t *phys_lookup = 0;
#endif
int err, map_err;
nr_contig_chunks = remaining_nr_contig_chunks = (int)window->nr_contig_chunks;
if ((map_err = micscif_map_window_pages(ep, window, false))) {
printk(KERN_ERR "%s %d map_err %d\n", __func__, __LINE__, map_err);
}
retry:
/* Now wait for the response */
err = wait_event_timeout(alloc->allocwq, alloc->state != OP_IN_PROGRESS, NODE_ALIVE_TIMEOUT);
if (!err && scifdev_alive(ep))
goto retry;
if (!err)
err = -ENODEV;
if (err > 0)
err = 0;
else
return err;
/* Bail out. The remote end rejected this request */
if (OP_FAILED == alloc->state)
return -ENOMEM;
if (map_err) {
printk(KERN_ERR "%s %d err %d\n", __func__, __LINE__, map_err);
msg.uop = SCIF_FREE_VIRT;
msg.src = ep->port;
msg.payload[0] = ep->remote_ep;
msg.payload[1] = (uint64_t)window->alloc_handle.vaddr;
msg.payload[2] = (uint64_t)window;
msg.payload[3] = SCIF_REGISTER;
if (!(err = micscif_nodeqp_send(ep->remote_dev, &msg, ep)))
err = -ENOTCONN;
else
err = map_err;
return err;
}
remote_window = scif_ioremap(alloc->phys_addr,
sizeof(*window), ep->remote_dev);
RMA_MAGIC(remote_window);
/* Compute the number of lookup entries. 21 == 2MB Shift */
nr_lookup = ALIGN(nr_contig_chunks * PAGE_SIZE, ((2) * 1024 * 1024)) >> 21;
#if defined(_MIC_SCIF_) && defined(CONFIG_ML1OM)
if (is_p2p_scifdev(ep->remote_dev))
phys_lookup = scif_ioremap(remote_window->temp_phys_addr_lookup.offset,
nr_lookup *
sizeof(*remote_window->temp_phys_addr_lookup.lookup),
ep->remote_dev);
#endif
dma_phys_lookup = scif_ioremap(remote_window->dma_addr_lookup.offset,
nr_lookup *
sizeof(*remote_window->dma_addr_lookup.lookup),
ep->remote_dev);
while (remaining_nr_contig_chunks) {
loop_nr_contig_chunks = min(remaining_nr_contig_chunks, (int)NR_PHYS_ADDR_IN_PAGE);
/* #1/2 - Copy physical addresses over to the remote side */
#if defined(_MIC_SCIF_) && defined(CONFIG_ML1OM)
/* If the remote dev is self or is any node except the host
* its OK to copy the bus address to the remote window
* in the case of the host (for KNF only) the bus address
* is generated at the time of mmap(..) into card memory
* and does not exist at this time
*/
/* Note:
* the phys_addr[] holds MIC address for remote cards
* -> GTT offset for the host (KNF)
* -> local address for the host (KNC)
* -> local address for loopback
* this is done in map_window_pages(..) except for GTT
* offset for KNF
*/
if (is_p2p_scifdev(ep->remote_dev)) {
tmp = scif_ioremap(phys_lookup[j],
loop_nr_contig_chunks * sizeof(*window->temp_phys_addr),
ep->remote_dev);
memcpy_toio(tmp, &window->temp_phys_addr[i],
loop_nr_contig_chunks * sizeof(*window->temp_phys_addr));
serializing_request(tmp);
smp_mb();
scif_iounmap(tmp, PAGE_SIZE, ep->remote_dev);
}
#endif
/* #2/2 - Copy DMA addresses (addresses that are fed into the DMA engine)
* We transfer bus addresses which are then converted into a MIC physical
* address on the remote side if it is a MIC, if the remote node is a host
* we transfer the MIC physical address
*/
tmp = scif_ioremap(
dma_phys_lookup[j],
loop_nr_contig_chunks * sizeof(*window->dma_addr),
ep->remote_dev);
#ifdef _MIC_SCIF_
if (is_p2p_scifdev(ep->remote_dev)) {
/* knf:
* send the address as mapped through the GTT (the remote node's
* base address for this node is already added in)
* knc:
* add remote node's base address for this node to convert it
* into a MIC address
*/
int m;
dma_addr_t dma_addr;
for (m = 0; m < loop_nr_contig_chunks; m++) {
#ifdef CONFIG_ML1OM
dma_addr = window->temp_phys_addr[i + m];
#else
dma_addr = window->dma_addr[i + m] +
ep->remote_dev->sd_base_addr;
#endif
writeq(dma_addr, &tmp[m]);
}
} else
/* Host node or loopback - transfer DMA addresses as is, this is
* the same as a MIC physical address (we use the dma_addr
* and not the phys_addr array since the phys_addr is only setup
* if there is a mmap() request from the host)
*/
memcpy_toio(tmp, &window->dma_addr[i],
loop_nr_contig_chunks * sizeof(*window->dma_addr));
#else
/* Transfer the physical address array - this is the MIC address
* as seen by the card
*/
memcpy_toio(tmp, &window->dma_addr[i],
loop_nr_contig_chunks * sizeof(*window->dma_addr));
#endif
remaining_nr_contig_chunks -= loop_nr_contig_chunks;
i += loop_nr_contig_chunks;
j++;
serializing_request(tmp);
smp_mb();
scif_iounmap(tmp, PAGE_SIZE, ep->remote_dev);
}
/* Prepare the remote window for the peer */
remote_window->peer_window = (uint64_t)window;
remote_window->offset = window->offset;
remote_window->prot = window->prot;
remote_window->nr_contig_chunks = nr_contig_chunks;
#ifdef _MIC_SCIF_
if (!ep->rma_info.proxy_dma_peer_phys)
ep->rma_info.proxy_dma_peer_phys = remote_window->proxy_dma_phys;
#endif
#if defined(_MIC_SCIF_) && defined(CONFIG_ML1OM)
if (is_p2p_scifdev(ep->remote_dev))
scif_iounmap(phys_lookup,
nr_lookup *
sizeof(*remote_window->temp_phys_addr_lookup.lookup),
ep->remote_dev);
#endif
scif_iounmap(dma_phys_lookup,
nr_lookup *
sizeof(*remote_window->dma_addr_lookup.lookup),
ep->remote_dev);
scif_iounmap(remote_window, sizeof(*remote_window), ep->remote_dev);
window->peer_window = (uint64_t)alloc->vaddr;
return err;
}
/**
* micscif_send_scif_register:
* @ep: end point
* @window: self registration window
*
* Send a SCIF_REGISTER message if EP is connected and wait for a
* SCIF_REGISTER_(N)ACK message else send a SCIF_FREE_VIRT
* message so that the peer can free its remote window allocated earlier.
*/
int micscif_send_scif_register(struct endpt *ep, struct reg_range_t *window)
{
int err = 0;
struct nodemsg msg;
msg.src = ep->port;
msg.payload[0] = ep->remote_ep;
msg.payload[1] = (uint64_t)window->alloc_handle.vaddr;
msg.payload[2] = (uint64_t)window;
if (SCIFEP_CONNECTED == ep->state) {
msg.uop = SCIF_REGISTER;
window->reg_state = OP_IN_PROGRESS;
if (!(err = micscif_nodeqp_send(ep->remote_dev, &msg, ep))) {
micscif_set_nr_pages(ep->remote_dev, window);
retry:
err = wait_event_timeout(window->regwq,
window->reg_state != OP_IN_PROGRESS, NODE_ALIVE_TIMEOUT);
if (!err && scifdev_alive(ep))
goto retry;
if (!err)
err = -ENODEV;
if (err > 0)
err = 0;
if (OP_FAILED == window->reg_state)
err = -ENOTCONN;
} else {
micscif_set_nr_pages(ep->remote_dev, window);
}
} else {
msg.uop = SCIF_FREE_VIRT;
msg.payload[3] = SCIF_REGISTER;
if (!(err = micscif_nodeqp_send(ep->remote_dev, &msg, ep)))
err = -ENOTCONN;
micscif_set_nr_pages(ep->remote_dev, window);
}
return err;
}
/**
* micscif_send_scif_unregister:
* @ep: end point
* @window: self registration window
*
* Send a SCIF_UNREGISTER message.
*/
int micscif_send_scif_unregister(struct endpt *ep, struct reg_range_t *window)
{
struct nodemsg msg;
RMA_MAGIC(window);
msg.uop = SCIF_UNREGISTER;
msg.src = ep->port;
msg.payload[0] = (uint64_t)window->alloc_handle.vaddr;
msg.payload[1] = (uint64_t)window;
return micscif_nodeqp_send(ep->remote_dev, &msg, ep);
}
/**
* micscif_get_window_offset:
* @epd: end point descriptor
* @flags: flags
* @offset: offset hint
* @len: length of range
* @out_offset: computed offset returned by reference.
*
* Compute/Claim a new offset for this EP. The callee is supposed to grab
* the RMA mutex before calling this API.
*/
int micscif_get_window_offset(struct endpt *ep, int flags,
uint64_t offset, size_t len, uint64_t *out_offset)
{
uint64_t computed_offset;
int err = 0;
might_sleep();
mutex_lock(&ep->rma_info.va_lock);
if (flags & SCIF_MAP_FIXED) {
computed_offset = va_gen_claim(&ep->rma_info.va_gen,
(uint64_t)offset, len);
if (INVALID_VA_GEN_ADDRESS == computed_offset)
err = -EADDRINUSE;
} else {
computed_offset = va_gen_alloc(&ep->rma_info.va_gen,
len, PAGE_SIZE);
if (INVALID_VA_GEN_ADDRESS == computed_offset)
err = -ENOMEM;
}
*out_offset = computed_offset;
mutex_unlock(&ep->rma_info.va_lock);
return err;
}
/**
* micscif_free_window_offset:
* @offset: offset hint
* @len: length of range
*
* Free offset for this EP. The callee is supposed to grab
* the RMA mutex before calling this API.
*/
void micscif_free_window_offset(struct endpt *ep,
uint64_t offset, size_t len)
{
mutex_lock(&ep->rma_info.va_lock);
va_gen_free(&ep->rma_info.va_gen, offset, len);
mutex_unlock(&ep->rma_info.va_lock);
}
/**
* scif_register_temp:
* @epd: End Point Descriptor.
* @addr: virtual address to/from which to copy
* @len: length of range to copy
* @out_offset: computed offset returned by reference.
* @out_window: allocated registered window returned by reference.
*
* Create a temporary registered window. The peer will not know about this
* window. This API is used for scif_vreadfrom()/scif_vwriteto() API's.
*/
static int
micscif_register_temp(scif_epd_t epd, void *addr, size_t len, int prot,
off_t *out_offset, struct reg_range_t **out_window)
{
struct endpt *ep = (struct endpt *)epd;
int err;
scif_pinned_pages_t pinned_pages;
size_t aligned_len;
aligned_len = ALIGN(len, PAGE_SIZE);
if ((err = __scif_pin_pages((void *)((uint64_t)addr &
PAGE_MASK),
aligned_len, &prot, 0, &pinned_pages)))
return err;
pinned_pages->prot = prot;
/* Compute the offset for this registration */
if ((err = micscif_get_window_offset(ep, 0, 0,
aligned_len, (uint64_t *)out_offset)))
goto error_unpin;
/* Allocate and prepare self registration window */
if (!(*out_window = micscif_create_window(ep, aligned_len >> PAGE_SHIFT,
*out_offset, true))) {
micscif_free_window_offset(ep, *out_offset, aligned_len);
err = -ENOMEM;
goto error_unpin;
}
(*out_window)->pinned_pages = pinned_pages;
(*out_window)->nr_pages = pinned_pages->nr_pages;
(*out_window)->nr_contig_chunks = pinned_pages->nr_contig_chunks;
(*out_window)->prot = pinned_pages->prot;
(*out_window)->va_for_temp = (void*)((uint64_t)addr & PAGE_MASK);
if ((err = micscif_map_window_pages(ep, *out_window, true))) {
/* Something went wrong! Rollback */
micscif_destroy_window(ep, *out_window);
*out_window = NULL;
} else
*out_offset |= ((uint64_t)addr & ~PAGE_MASK);
return err;
error_unpin:
if (err)
printk(KERN_ERR "%s %d err %d\n", __func__, __LINE__, err);
scif_unpin_pages(pinned_pages);
return err;
}
/**
* micscif_rma_completion_cb:
* @data: RMA cookie
*
* RMA interrupt completion callback.
*/
void micscif_rma_completion_cb(uint64_t data)
{
struct dma_completion_cb *comp_cb = (struct dma_completion_cb *)data;
#ifndef _MIC_SCIF_
struct pci_dev *pdev;
#endif
/* Free DMA Completion CB. */
if (comp_cb && comp_cb->temp_buf) {
if (comp_cb->dst_window) {
micscif_rma_local_cpu_copy(comp_cb->dst_offset,
comp_cb->dst_window, comp_cb->temp_buf + comp_cb->header_padding,
comp_cb->len, false);
}
#ifndef _MIC_SCIF_
micscif_pci_dev(comp_cb->remote_node, &pdev);
mic_ctx_unmap_single(get_per_dev_ctx(comp_cb->remote_node - 1),
comp_cb->temp_phys, KMEM_UNALIGNED_BUF_SIZE);
#endif
if (comp_cb->is_cache)
micscif_kmem_cache_free(comp_cb->temp_buf_to_free);
else
kfree(comp_cb->temp_buf_to_free);
}
kfree(comp_cb);
}
static void __micscif_rma_destroy_tcw_ep(struct endpt *ep);
static
bool micscif_rma_tc_can_cache(struct endpt *ep, size_t cur_bytes)
{
if ((cur_bytes >> PAGE_SHIFT) > ms_info.mi_rma_tc_limit)
return false;
if ((atomic_read(&ep->rma_info.tcw_total_pages)
+ (cur_bytes >> PAGE_SHIFT)) >
ms_info.mi_rma_tc_limit) {
printk(KERN_ALERT "%s %d total=%d, current=%zu reached max\n",
__func__, __LINE__,
atomic_read(&ep->rma_info.tcw_total_pages),
(1 + (cur_bytes >> PAGE_SHIFT)));
micscif_rma_destroy_tcw_invalid(&ms_info.mi_rma_tc);
__micscif_rma_destroy_tcw_ep(ep);
}
return true;
}
/**
* micscif_rma_copy:
* @epd: end point descriptor.
* @loffset: offset in local registered address space to/from which to copy
* @addr: user virtual address to/from which to copy
* @len: length of range to copy
* @roffset: offset in remote registered address space to/from which to copy
* @flags: flags
* @dir: LOCAL->REMOTE or vice versa.
*
* Validate parameters, check if src/dst registered ranges requested for copy
* are valid and initiate either CPU or DMA copy.
*/
int micscif_rma_copy(scif_epd_t epd, off_t loffset, void *addr, size_t len,
off_t roffset, int flags, enum rma_direction dir, bool last_chunk)
{
struct endpt *ep = (struct endpt *)epd;
struct micscif_rma_req remote_req;
struct micscif_rma_req req;
struct reg_range_t *window = NULL;
struct reg_range_t *remote_window = NULL;
struct mic_copy_work copy_work;
bool loopback;
int err = 0;
struct dma_channel *chan;
struct rma_mmu_notifier *mmn = NULL;
bool insert_window = false;
bool cache = false;
if ((err = verify_epd(ep)))
return err;
if (flags && !(flags & (SCIF_RMA_USECPU | SCIF_RMA_USECACHE | SCIF_RMA_SYNC | SCIF_RMA_ORDERED)))
return -EINVAL;
if (!len)
return -EINVAL;
loopback = is_self_scifdev(ep->remote_dev) ? true : false;
copy_work.fence_type = ((flags & SCIF_RMA_SYNC) && last_chunk) ? DO_DMA_POLLING : 0;
copy_work.ordered = !!((flags & SCIF_RMA_ORDERED) && last_chunk);
#ifdef CONFIG_MMU_NOTIFIER
if (!mic_reg_cache_enable)
flags &= ~SCIF_RMA_USECACHE;
#else
flags &= ~SCIF_RMA_USECACHE;
#endif
#ifndef _MIC_SCIF_
#ifdef CONFIG_ML1OM
/* Use DMA Copies even if CPU copy is requested on KNF MIC from Host */
if (flags & SCIF_RMA_USECPU) {
flags &= ~SCIF_RMA_USECPU;
if (last_chunk)
copy_work.fence_type = DO_DMA_POLLING;
}
#endif
/* Use CPU for Host<->Host Copies */
if (loopback) {
flags |= SCIF_RMA_USECPU;
copy_work.fence_type = 0x0;
}
#endif
cache = flags & SCIF_RMA_USECACHE;
/* Trying to wrap around */
if ((loffset && (loffset + (off_t)len < loffset)) ||
(roffset + (off_t)len < roffset))
return -EINVAL;
remote_req.out_window = &remote_window;
remote_req.offset = roffset;
remote_req.nr_bytes = len;
/*
* If transfer is from local to remote then the remote window
* must be writeable and vice versa.
*/
remote_req.prot = LOCAL_TO_REMOTE == dir ? VM_WRITE : VM_READ;
remote_req.type = WINDOW_PARTIAL;
remote_req.head = &ep->rma_info.remote_reg_list;
#ifdef CONFIG_MMU_NOTIFIER
if (addr && cache) {
mutex_lock(&ep->rma_info.mmn_lock);
mmn = find_mmu_notifier(current->mm, &ep->rma_info);
if (!mmn) {
mmn = kzalloc(sizeof(*mmn), GFP_KERNEL);
if (!mmn) {
mutex_unlock(&ep->rma_info.mmn_lock);
return -ENOMEM;
}
init_mmu_notifier(mmn, current->mm, ep);
if (mmu_notifier_register(&mmn->ep_mmu_notifier, current->mm)) {
mutex_unlock(&ep->rma_info.mmn_lock);
kfree(mmn);
return -EBUSY;
}
#ifdef RMA_DEBUG
atomic_long_add_return(1, &ms_info.mmu_notif_cnt);
#endif
list_add(&mmn->list_member, &ep->rma_info.mmn_list);
}
mutex_unlock(&ep->rma_info.mmn_lock);
}
#endif
micscif_inc_node_refcnt(ep->remote_dev, 1);
#ifdef _MIC_SCIF_
if (!(flags & SCIF_RMA_USECPU)) {
/*
* Proxy the DMA only for P2P reads with transfer size
* greater than proxy DMA threshold. scif_vreadfrom(..)
* and scif_vwriteto(..) is not supported since the peer
* does not have the page lists required to perform the
* proxy DMA.
*/
if (ep->remote_dev->sd_proxy_dma_reads &&
!addr && dir == REMOTE_TO_LOCAL &&
ep->rma_info.proxy_dma_va &&
len >= ms_info.mi_proxy_dma_threshold) {
copy_work.len = len;
copy_work.src_offset = roffset;
copy_work.dst_offset = loffset;
/* Fall through if there were errors */
if (!(err = micscif_proxy_dma(epd, &copy_work)))
goto error;
}
}
#endif
mutex_lock(&ep->rma_info.rma_lock);
if (addr) {
req.out_window = &window;
req.nr_bytes = ALIGN(len + ((uint64_t)addr & ~PAGE_MASK), PAGE_SIZE);
if (mmn)
req.head = &mmn->tc_reg_list;
req.va_for_temp = (void*)((uint64_t)addr & PAGE_MASK);
req.prot = (LOCAL_TO_REMOTE == dir ? VM_READ : VM_WRITE | VM_READ);
/* Does a valid local window exist? */
pr_debug("%s %d req.va_for_temp %p addr %p req.nr_bytes 0x%lx len 0x%lx\n",
__func__, __LINE__, req.va_for_temp, addr, req.nr_bytes, len);
spin_lock(&ep->rma_info.tc_lock);
if (!mmn || (err = micscif_query_tcw(ep, &req))) {
pr_debug("%s %d err %d req.va_for_temp %p addr %p req.nr_bytes 0x%lx len 0x%lx\n",
__func__, __LINE__, err, req.va_for_temp, addr, req.nr_bytes, len);
spin_unlock(&ep->rma_info.tc_lock);
mutex_unlock(&ep->rma_info.rma_lock);
if (cache)
if (!micscif_rma_tc_can_cache(ep, req.nr_bytes))
cache = false;
if ((err = micscif_register_temp(epd, req.va_for_temp, req.nr_bytes,
req.prot,
&loffset, &window))) {
goto error;
}
mutex_lock(&ep->rma_info.rma_lock);
pr_debug("New temp window created addr %p\n", addr);
if (cache) {
atomic_inc(&ep->rma_info.tcw_refcount);
atomic_add_return((int32_t)window->nr_pages, &ep->rma_info.tcw_total_pages);
if (mmn) {
spin_lock(&ep->rma_info.tc_lock);
micscif_insert_tcw(window, &mmn->tc_reg_list);
spin_unlock(&ep->rma_info.tc_lock);
}
}
insert_window = true;
} else {
spin_unlock(&ep->rma_info.tc_lock);
pr_debug("window found for addr %p\n", addr);
BUG_ON(window->va_for_temp > addr);
}
loffset = window->offset + ((uint64_t)addr - (uint64_t)window->va_for_temp);
pr_debug("%s %d addr %p loffset 0x%lx window->nr_pages 0x%llx"
" window->va_for_temp %p\n", __func__, __LINE__,
addr, loffset, window->nr_pages, window->va_for_temp);
RMA_MAGIC(window);
}
/* Does a valid remote window exist? */
if ((err = micscif_query_window(&remote_req))) {
pr_debug("%s %d err %d roffset 0x%lx len 0x%lx\n",
__func__, __LINE__, err, roffset, len);
mutex_unlock(&ep->rma_info.rma_lock);
goto error;
}
RMA_MAGIC(remote_window);
if (!addr) {
req.out_window = &window;
req.offset = loffset;
/*
* If transfer is from local to remote then the self window
* must be readable and vice versa.
*/
req.prot = LOCAL_TO_REMOTE == dir ? VM_READ : VM_WRITE;
req.nr_bytes = len;
req.type = WINDOW_PARTIAL;
req.head = &ep->rma_info.reg_list;
/* Does a valid local window exist? */
if ((err = micscif_query_window(&req))) {
printk(KERN_ERR "%s %d err %d\n", __func__, __LINE__, err);
mutex_unlock(&ep->rma_info.rma_lock);
goto error;
}
RMA_MAGIC(window);
}
/*
* Preprare copy_work for submitting work to the DMA kernel thread
* or CPU copy routine.
*/
copy_work.len = len;
copy_work.loopback = loopback;
copy_work.remote_dev = ep->remote_dev;
copy_work.dma_chan_released = false;
if (LOCAL_TO_REMOTE == dir) {
copy_work.src_offset = loffset;
copy_work.src_window = window;
copy_work.dst_offset = roffset;
copy_work.dst_window = remote_window;
} else {
copy_work.src_offset = roffset;
copy_work.src_window = remote_window;
copy_work.dst_offset = loffset;
copy_work.dst_window = window;
}
if (!(flags & SCIF_RMA_USECPU)) {
chan = ep->rma_info.dma_chan;
if ((err = request_dma_channel(chan))) {
mutex_unlock(&ep->rma_info.rma_lock);
goto error;
}
err = micscif_rma_list_dma_copy_wrapper(epd, &copy_work,
chan, loffset);
if (!copy_work.dma_chan_released)
free_dma_channel(chan);
}
if (flags & SCIF_RMA_USECPU) {
/* Initiate synchronous CPU copy */
micscif_rma_list_cpu_copy(&copy_work);
}
if (insert_window && !cache) {
atomic_inc(&ep->rma_info.tw_refcount);
atomic_add_return((int32_t)window->nr_pages, &ep->rma_info.tw_total_pages);
}
mutex_unlock(&ep->rma_info.rma_lock);
if (last_chunk) {
if (DO_DMA_POLLING == copy_work.fence_type)
err = drain_dma_poll(ep->rma_info.dma_chan);
else if (DO_DMA_INTR == copy_work.fence_type)
err = drain_dma_intr(ep->rma_info.dma_chan);
}
micscif_dec_node_refcnt(ep->remote_dev, 1);
if (insert_window && !cache)
micscif_queue_for_cleanup(window, &ms_info.mi_rma);
return err;
error:
if (err) {
if (addr && window && !cache)
micscif_destroy_window(ep, window);
printk(KERN_ERR "%s %d err %d len 0x%lx\n", __func__, __LINE__, err, len);
}
micscif_dec_node_refcnt(ep->remote_dev, 1);
return err;
}
/**
* micscif_send_fence_mark:
* @epd: end point descriptor.
* @out_mark: Output DMA mark reported by peer.
*
* Send a remote fence mark request.
*/
int micscif_send_fence_mark(scif_epd_t epd, int *out_mark)
{
int err;
struct nodemsg msg;
struct fence_info *fence_req;
struct endpt *ep = (struct endpt *)epd;
if (!(fence_req = kmalloc(sizeof(*fence_req), GFP_KERNEL))) {
err = -ENOMEM;
goto error;
}
fence_req->state = OP_IN_PROGRESS;
init_waitqueue_head(&fence_req->wq);
msg.src = ep->port;
msg.uop = SCIF_MARK;
msg.payload[0] = ep->remote_ep;
msg.payload[1] = (uint64_t)fence_req;
if ((err = micscif_nodeqp_send(ep->remote_dev, &msg, ep)))
goto error;
retry:
err = wait_event_timeout(fence_req->wq,
(OP_IN_PROGRESS != fence_req->state), NODE_ALIVE_TIMEOUT);
if (!err && scifdev_alive(ep))
goto retry;
if (!err)
err = -ENODEV;
if (err > 0)
err = 0;
if (err < 0) {
mutex_lock(&ep->rma_info.rma_lock);
if (OP_IN_PROGRESS == fence_req->state)
fence_req->state = OP_FAILED;
mutex_unlock(&ep->rma_info.rma_lock);
}
if (OP_COMPLETED == fence_req->state)
*out_mark = SCIF_REMOTE_FENCE | fence_req->dma_mark;
if (OP_FAILED == fence_req->state && !err)
err = -ENOMEM;
mutex_lock(&ep->rma_info.rma_lock);
mutex_unlock(&ep->rma_info.rma_lock);
kfree(fence_req);
error:
return err;
}
/**
* micscif_send_fence_wait:
* @epd: end point descriptor.
* @mark: DMA mark to wait for.
*
* Send a remote fence wait request.
*/
int micscif_send_fence_wait(scif_epd_t epd, int mark)
{
int err;
struct nodemsg msg;
struct fence_info *fence_req;
struct endpt *ep = (struct endpt *)epd;
if (!(fence_req = kmalloc(sizeof(*fence_req), GFP_KERNEL))) {
err = -ENOMEM;
goto error;
}
fence_req->state = OP_IN_PROGRESS;
init_waitqueue_head(&fence_req->wq);
msg.src = ep->port;
msg.uop = SCIF_WAIT;
msg.payload[0] = ep->remote_ep;
msg.payload[1] = (uint64_t)fence_req;
msg.payload[2] = mark;
if ((err = micscif_nodeqp_send(ep->remote_dev, &msg, ep)))
goto error;
retry:
err = wait_event_timeout(fence_req->wq,
(OP_IN_PROGRESS != fence_req->state), NODE_ALIVE_TIMEOUT);
if (!err && scifdev_alive(ep))
goto retry;
if (!err)
err = -ENODEV;
if (err > 0)
err = 0;
if (err < 0) {
mutex_lock(&ep->rma_info.rma_lock);
if (OP_IN_PROGRESS == fence_req->state)
fence_req->state = OP_FAILED;
mutex_unlock(&ep->rma_info.rma_lock);
}
if (OP_FAILED == fence_req->state && !err)
err = -ENOMEM;
mutex_lock(&ep->rma_info.rma_lock);
mutex_unlock(&ep->rma_info.rma_lock);
kfree(fence_req);
error:
return err;
}
/**
* micscif_send_fence_signal:
* @epd - endpoint descriptor
* @loff - local offset
* @lval - local value to write to loffset
* @roff - remote offset
* @rval - remote value to write to roffset
* @flags - flags
*
* Sends a remote fence signal request
*/
int micscif_send_fence_signal(scif_epd_t epd, off_t roff, uint64_t rval,
off_t loff, uint64_t lval, int flags)
{
int err = 0;
struct nodemsg msg;
struct fence_info *fence_req;
struct endpt *ep = (struct endpt *)epd;
if (!(fence_req = kmalloc(sizeof(*fence_req), GFP_KERNEL))) {
err = -ENOMEM;
goto error;
}
fence_req->state = OP_IN_PROGRESS;
init_waitqueue_head(&fence_req->wq);
msg.src = ep->port;
if (flags & SCIF_SIGNAL_LOCAL) {
msg.uop = SCIF_SIG_LOCAL;
msg.payload[0] = ep->remote_ep;
msg.payload[1] = roff;
msg.payload[2] = rval;
msg.payload[3] = (uint64_t)fence_req;
if ((err = micscif_nodeqp_send(ep->remote_dev, &msg, ep)))
goto error_free;
retry1:
err = wait_event_timeout(fence_req->wq,
(OP_IN_PROGRESS != fence_req->state), NODE_ALIVE_TIMEOUT);
if (!err && scifdev_alive(ep))
goto retry1;
if (!err)
err = -ENODEV;
if (err > 0)
err = 0;
if (err < 0) {
mutex_lock(&ep->rma_info.rma_lock);
if (OP_IN_PROGRESS == fence_req->state)
fence_req->state = OP_FAILED;
mutex_unlock(&ep->rma_info.rma_lock);
}
if (OP_FAILED == fence_req->state && !err) {
err = -ENXIO;
goto error_free;
}
}
fence_req->state = OP_IN_PROGRESS;
if (flags & SCIF_SIGNAL_REMOTE) {
msg.uop = SCIF_SIG_REMOTE;
msg.payload[0] = ep->remote_ep;
msg.payload[1] = loff;
msg.payload[2] = lval;
msg.payload[3] = (uint64_t)fence_req;
if ((err = micscif_nodeqp_send(ep->remote_dev, &msg, ep)))
goto error_free;
retry2:
err = wait_event_timeout(fence_req->wq,
(OP_IN_PROGRESS != fence_req->state), NODE_ALIVE_TIMEOUT);
if (!err && scifdev_alive(ep))
goto retry2;
if (!err)
err = -ENODEV;
if (err > 0)
err = 0;
if (err < 0) {
mutex_lock(&ep->rma_info.rma_lock);
if (OP_IN_PROGRESS == fence_req->state)
fence_req->state = OP_FAILED;
mutex_unlock(&ep->rma_info.rma_lock);
}
if (OP_FAILED == fence_req->state && !err) {
err = -ENXIO;
goto error_free;
}
}
error_free:
mutex_lock(&ep->rma_info.rma_lock);
mutex_unlock(&ep->rma_info.rma_lock);
kfree(fence_req);
error:
return err;
}
/*
* micscif_fence_mark:
*
* @epd - endpoint descriptor
* Set up a mark for this endpoint and return the value of the mark.
*/
int micscif_fence_mark(scif_epd_t epd)
{
int mark = 0;
struct endpt *ep = (struct endpt *)epd;
struct dma_channel *chan = ep->rma_info.dma_chan;
if ((mark = request_dma_channel(chan)))
goto error;
mark = program_dma_mark(chan);
free_dma_channel(chan);
error:
return mark;
}
/**
* micscif_rma_destroy_temp_windows:
*
* This routine destroys temporary registered windows created
* by scif_vreadfrom() and scif_vwriteto().
*/
void micscif_rma_destroy_temp_windows(void)
{
struct list_head *item, *tmp;
struct reg_range_t *window;
struct endpt *ep;
struct dma_channel *chan;
might_sleep();
restart:
spin_lock(&ms_info.mi_rmalock);
list_for_each_safe(item, tmp, &ms_info.mi_rma) {
window = list_entry(item,
struct reg_range_t, list_member);
ep = (struct endpt *)window->ep;
chan = ep->rma_info.dma_chan;
list_del(&window->list_member);
spin_unlock(&ms_info.mi_rmalock);
micscif_inc_node_refcnt(ep->remote_dev, 1);
if (!chan ||
!scifdev_alive(ep) ||
(!is_current_dma_mark(chan, window->dma_mark) &&
is_dma_mark_processed(chan, window->dma_mark)) ||
!drain_dma_intr(chan)) {
micscif_dec_node_refcnt(ep->remote_dev, 1);
/* Remove window from global list */
window->unreg_state = OP_COMPLETED;
} else {
micscif_dec_node_refcnt(ep->remote_dev, 1);
/* DMA engine hung ?? */
printk(KERN_ERR "%s %d DMA channel %d hung ep->state %d "
"window->dma_mark 0x%x channel_mark 0x%x\n",
__func__, __LINE__, get_chan_num(chan),
ep->sd_state, window->dma_mark, get_dma_mark(chan));
WARN_ON(1);
micscif_queue_for_cleanup(window, &ms_info.mi_rma);
goto restart;
}
if (OP_COMPLETED == window->unreg_state) {
BUG_ON(atomic_sub_return((int32_t)window->nr_pages,
&ep->rma_info.tw_total_pages) < 0);
if (RMA_WINDOW_SELF == window->type)
micscif_destroy_window(ep, window);
else
micscif_destroy_remote_window(ep, window);
BUG_ON(atomic_dec_return(
&ep->rma_info.tw_refcount) < 0);
}
goto restart;
}
spin_unlock(&ms_info.mi_rmalock);
}
/**
* micscif_rma_destroy_tcw:
*
* This routine destroys temporary registered windows created
* by scif_vreadfrom() and scif_vwriteto().
*/
static
void __micscif_rma_destroy_tcw(struct rma_mmu_notifier *mmn,
struct endpt *ep, bool inrange,
uint64_t start, uint64_t len)
{
struct list_head *item, *tmp;
struct reg_range_t *window;
uint64_t start_va, end_va;
uint64_t end = start + len;
list_for_each_safe(item, tmp, &mmn->tc_reg_list) {
window = list_entry(item,
struct reg_range_t, list_member);
ep = (struct endpt *)window->ep;
if (inrange) {
if (0 == len)
break;
start_va = (uint64_t)window->va_for_temp;
end_va = start_va+ (window->nr_pages << PAGE_SHIFT);
if (start < start_va) {
if (end <= start_va) {
break;
} else {
}
} else {
if (start >= end_va) {
continue;
} else {
}
}
}
__micscif_rma_destroy_tcw_helper(window);
}
}
static inline
void micscif_rma_destroy_tcw(struct rma_mmu_notifier *mmn,
struct endpt *ep, bool inrange,
uint64_t start, uint64_t len)
{
unsigned long sflags;
spin_lock_irqsave(&ep->rma_info.tc_lock, sflags);
__micscif_rma_destroy_tcw(mmn, ep, inrange, start, len);
spin_unlock_irqrestore(&ep->rma_info.tc_lock, sflags);
}
static void __micscif_rma_destroy_tcw_ep(struct endpt *ep)
{
struct list_head *item, *tmp;
struct rma_mmu_notifier *mmn;
spin_lock(&ep->rma_info.tc_lock);
list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {
mmn = list_entry(item,
struct rma_mmu_notifier, list_member);
__micscif_rma_destroy_tcw(mmn, ep, false, 0, 0);
}
spin_unlock(&ep->rma_info.tc_lock);
}
void micscif_rma_destroy_tcw_ep(struct endpt *ep)
{
struct list_head *item, *tmp;
struct rma_mmu_notifier *mmn;
list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {
mmn = list_entry(item,
struct rma_mmu_notifier, list_member);
micscif_rma_destroy_tcw(mmn, ep, false, 0, 0);
}
}
/**
* micscif_rma_destroy_tcw:
*
* This routine destroys temporary registered windows created
* by scif_vreadfrom() and scif_vwriteto().
*/
void micscif_rma_destroy_tcw_invalid(struct list_head *list)
{
struct list_head *item, *tmp;
struct reg_range_t *window;
struct endpt *ep;
struct dma_channel *chan;
might_sleep();
restart:
spin_lock(&ms_info.mi_rmalock);
list_for_each_safe(item, tmp, list) {
window = list_entry(item,
struct reg_range_t, list_member);
ep = (struct endpt *)window->ep;
chan = ep->rma_info.dma_chan;
list_del(&window->list_member);
spin_unlock(&ms_info.mi_rmalock);
micscif_inc_node_refcnt(ep->remote_dev, 1);
mutex_lock(&ep->rma_info.rma_lock);
if (!chan ||
!scifdev_alive(ep) ||
(!is_current_dma_mark(chan, window->dma_mark) &&
is_dma_mark_processed(chan, window->dma_mark)) ||
!drain_dma_intr(chan)) {
micscif_dec_node_refcnt(ep->remote_dev, 1);
BUG_ON(atomic_sub_return((int32_t)window->nr_pages,
&ep->rma_info.tcw_total_pages) < 0);
micscif_destroy_window(ep, window);
BUG_ON(atomic_dec_return(
&ep->rma_info.tcw_refcount) < 0);
} else {
/* DMA engine hung ?? */
printk(KERN_ERR "%s %d DMA channel %d hung ep->state %d "
"window->dma_mark 0x%x channel_mark 0x%x\n",
__func__, __LINE__, get_chan_num(chan),
ep->sd_state, window->dma_mark, get_dma_mark(chan));
WARN_ON(1);
mutex_unlock(&ep->rma_info.rma_lock);
micscif_dec_node_refcnt(ep->remote_dev, 1);
micscif_queue_for_cleanup(window, &ms_info.mi_rma);
goto restart;
}
mutex_unlock(&ep->rma_info.rma_lock);
goto restart;
}
spin_unlock(&ms_info.mi_rmalock);
}
/**
* micscif_rma_handle_remote_fences:
*
* This routine services remote fence requests.
*/
void micscif_rma_handle_remote_fences(void)
{
struct list_head *item, *tmp;
struct remote_fence_info *fence;
struct endpt *ep;
int mark;
might_sleep();
mutex_lock(&ms_info.mi_fencelock);
list_for_each_safe(item, tmp, &ms_info.mi_fence) {
fence = list_entry(item,
struct remote_fence_info, list_member);
/* Remove fence from global list */
list_del(&fence->list_member);
/* Initiate the fence operation */
ep = (struct endpt *)fence->msg.payload[0];
mark = (int)fence->msg.payload[2];
BUG_ON(!(mark & SCIF_REMOTE_FENCE));
if (dma_mark_wait(ep->rma_info.dma_chan,
mark & ~SCIF_REMOTE_FENCE, false)) {
printk(KERN_ERR "%s %d err\n", __func__, __LINE__);
fence->msg.uop = SCIF_WAIT_NACK;
} else {
fence->msg.uop = SCIF_WAIT_ACK;
}
micscif_inc_node_refcnt(ep->remote_dev, 1);
fence->msg.payload[0] = ep->remote_ep;
/* No error handling for Notification messages. */
micscif_nodeqp_send(ep->remote_dev, &fence->msg, ep);
micscif_dec_node_refcnt(ep->remote_dev, 1);
kfree(fence);
/*
* Decrement ref count and wake up
* any thread blocked in the EP close routine waiting
* for all such remote fence requests to complete.
*/
ep->rma_info.fence_refcount--;
wake_up(&ep->rma_info.fence_wq);
}
mutex_unlock(&ms_info.mi_fencelock);
}
#ifdef CONFIG_MMU_NOTIFIER
void micscif_mmu_notif_handler(struct work_struct *work)
{
struct list_head *pos, *tmpq;
struct endpt *ep;
restart:
micscif_rma_destroy_tcw_invalid(&ms_info.mi_rma_tc);
spin_lock(&ms_info.mi_rmalock);
list_for_each_safe(pos, tmpq, &ms_info.mi_mmu_notif_cleanup) {
ep = list_entry(pos, struct endpt, mmu_list);
list_del(&ep->mmu_list);
spin_unlock(&ms_info.mi_rmalock);
BUG_ON(list_empty(&ep->rma_info.mmn_list));
micscif_rma_destroy_tcw_ep(ep);
ep_unregister_mmu_notifier(ep);
queue_work(ms_info.mi_misc_wq, &ms_info.mi_misc_work);
goto restart;
}
spin_unlock(&ms_info.mi_rmalock);
}
#endif
/**
* micscif_reserve_dma_chan:
* @ep: Endpoint Descriptor.
*
* This routine reserves a DMA channel for a particular
* endpoint. All DMA transfers for an endpoint are always
* programmed on the same DMA channel.
*/
int micscif_reserve_dma_chan(struct endpt *ep)
{
int err = 0;
#ifndef _MIC_SCIF_
/*
* Host Loopback cannot use DMA by design and hence
* reserving DMA channels is a nop.
*/
if (is_self_scifdev(ep->remote_dev))
return 0;
#endif
mutex_lock(&ep->rma_info.rma_lock);
if (!ep->rma_info.dma_chan) {
struct dma_channel **chan = &ep->rma_info.dma_chan;
unsigned long ts = jiffies;
#ifndef _MIC_SCIF_
mic_ctx_t *mic_ctx =
get_per_dev_ctx(ep->remote_dev->sd_node - 1);
BUG_ON(!ep->remote_dev->sd_node);
#endif
while (true) {
if (!(err = allocate_dma_channel((struct mic_dma_ctx_t *)
#ifdef _MIC_SCIF_
mic_dma_handle,
#else
mic_ctx->dma_handle,
#endif
chan)))
break;
schedule();
if (time_after(jiffies,
ts + NODE_ALIVE_TIMEOUT)) {
err = -EBUSY;
goto error;
}
}
mic_dma_thread_free_chan(*chan);
}
error:
mutex_unlock(&ep->rma_info.rma_lock);
return err;
}
/*
* micscif_prog_signal:
* @epd - Endpoint Descriptor
* @offset - registered address
* @val - Value to be programmed in SUD.
* @type - Type of the window.
*
* Program a status update descriptor adter ensuring that the offset
* provided is indeed valid.
*/
int micscif_prog_signal(scif_epd_t epd, off_t offset, uint64_t val,
enum rma_window_type type)
{
struct endpt *ep = (struct endpt *)epd;
struct dma_channel *chan = ep->rma_info.dma_chan;
struct reg_range_t *window = NULL;
struct micscif_rma_req req;
int err;
dma_addr_t phys;
mutex_lock(&ep->rma_info.rma_lock);
req.out_window = &window;
req.offset = offset;
req.nr_bytes = sizeof(uint64_t);
req.prot = SCIF_PROT_WRITE;
req.type = WINDOW_SINGLE;
if (RMA_WINDOW_SELF == type)
req.head = &ep->rma_info.reg_list;
else
req.head = &ep->rma_info.remote_reg_list;
/* Does a valid window exist? */
if ((err = micscif_query_window(&req))) {
printk(KERN_ERR "%s %d err %d\n",
__func__, __LINE__, err);
goto unlock_ret;
}
RMA_MAGIC(window);
#ifndef _MIC_SCIF_
if (unlikely(is_self_scifdev(ep->remote_dev))) {
void *dst_virt;
if (RMA_WINDOW_SELF == type)
dst_virt = get_local_va(offset, window,
sizeof(uint32_t));
else {
struct page **pages = ((struct reg_range_t *)
(window->peer_window))->pinned_pages->pages;
int page_nr = (int) ( (offset - window->offset) >> PAGE_SHIFT );
off_t page_off = offset & ~PAGE_MASK;
dst_virt = (void *)((uint64_t)phys_to_virt(page_to_phys(
pages[page_nr])) | page_off);
}
*(uint64_t*)dst_virt = val;
goto unlock_ret;
}
#endif
phys = micscif_get_dma_addr(window, offset, NULL, NULL, NULL);
if ((err = request_dma_channel(chan)))
goto unlock_ret;
err = do_status_update(chan, phys, val);
free_dma_channel(chan);
unlock_ret:
mutex_unlock(&ep->rma_info.rma_lock);
return err;
}
/*
* __micscif_kill_apps_with_mmaps:
* @ep - The SCIF endpoint
*
* Kill the applications which have valid remote memory mappings
* created via scif_mmap(..).
*/
static void __micscif_kill_apps_with_mmaps(struct endpt *ep)
{
struct list_head *item;
struct rma_task_info *info;
spin_lock(&ep->lock);
list_for_each(item, &ep->rma_info.task_list) {
info = list_entry(item, struct rma_task_info, list_member);
kill_pid(info->pid, SIGKILL, 1);
pr_debug("%s ep %p pid %p ref %d\n",
__func__, ep, info->pid, info->ref_count);
}
spin_unlock(&ep->lock);
}
/*
* _micscif_kill_apps_with_mmaps:
* @node - remote node id.
* @head - head of the list of endpoints to kill.
*
* Traverse the list of endpoints for a particular remote node and
* kill applications with valid remote memory mappings.
*/
static void _micscif_kill_apps_with_mmaps(int node, struct list_head *head)
{
struct endpt *ep;
unsigned long sflags;
struct list_head *item;
spin_lock_irqsave(&ms_info.mi_connlock, sflags);
list_for_each(item, head) {
ep = list_entry(item, struct endpt, list);
if (ep->remote_dev->sd_node == node)
__micscif_kill_apps_with_mmaps(ep);
}
spin_unlock_irqrestore(&ms_info.mi_connlock, sflags);
}
/*
* micscif_kill_apps_with_mmaps:
* @node - remote node id.
*
* Wrapper for killing applications with valid remote memory mappings
* for a particular node. This API is called by peer nodes as part of
* handling a lost node.
*/
void micscif_kill_apps_with_mmaps(int node)
{
_micscif_kill_apps_with_mmaps(node, &ms_info.mi_connected);
_micscif_kill_apps_with_mmaps(node, &ms_info.mi_disconnected);
}
/*
* micscif_query_apps_with_mmaps:
* @node - remote node id.
* @head - head of the list of endpoints to query.
*
* Query if any applications for a remote node have valid remote memory
* mappings.
*/
static bool micscif_query_apps_with_mmaps(int node, struct list_head *head)
{
struct endpt *ep;
unsigned long sflags;
struct list_head *item;
bool ret = false;
spin_lock_irqsave(&ms_info.mi_connlock, sflags);
list_for_each(item, head) {
ep = list_entry(item, struct endpt, list);
if (ep->remote_dev->sd_node == node &&
!list_empty(&ep->rma_info.task_list)) {
ret = true;
break;
}
}
spin_unlock_irqrestore(&ms_info.mi_connlock, sflags);
return ret;
}
/*
* micscif_rma_do_apps_have_mmaps:
* @node - remote node id.
*
* Wrapper for querying if any applications have remote memory mappings
* for a particular node.
*/
bool micscif_rma_do_apps_have_mmaps(int node)
{
return (micscif_query_apps_with_mmaps(node, &ms_info.mi_connected) ||
micscif_query_apps_with_mmaps(node, &ms_info.mi_disconnected));
}
/*
* __micscif_cleanup_rma_for_zombies:
* @ep - The SCIF endpoint
*
* This API is only called while handling a lost node:
* a) Remote node is dead.
* b) All endpoints with remote memory mappings have been killed.
* So we can traverse the remote_reg_list without any locks. Since
* the window has not yet been unregistered we can drop the ref count
* and queue it to the cleanup thread.
*/
static void __micscif_cleanup_rma_for_zombies(struct endpt *ep)
{
struct list_head *pos, *tmp;
struct reg_range_t *window;
list_for_each_safe(pos, tmp, &ep->rma_info.remote_reg_list) {
window = list_entry(pos, struct reg_range_t, list_member);
/* If unregistration is complete then why is it on the list? */
WARN_ON(window->unreg_state == OP_COMPLETED);
if (window->ref_count)
put_window_ref_count(window, window->nr_pages);
if (!window->ref_count) {
atomic_inc(&ep->rma_info.tw_refcount);
atomic_add_return((int32_t)window->nr_pages,
&ep->rma_info.tw_total_pages);
list_del(&window->list_member);
micscif_queue_for_cleanup(window, &ms_info.mi_rma);
}
}
}
/*
* micscif_cleanup_rma_for_zombies:
* @node - remote node id.
*
* Cleanup remote registration lists for zombie endpoints.
*/
void micscif_cleanup_rma_for_zombies(int node)
{
struct endpt *ep;
unsigned long sflags;
struct list_head *item;
spin_lock_irqsave(&ms_info.mi_eplock, sflags);
list_for_each(item, &ms_info.mi_zombie) {
ep = list_entry(item, struct endpt, list);
if (ep->remote_dev && ep->remote_dev->sd_node == node) {
/*
* If the zombie endpoint remote node matches the lost
* node then the scifdev should not be alive.
*/
WARN_ON(scifdev_alive(ep));
__micscif_cleanup_rma_for_zombies(ep);
}
}
spin_unlock_irqrestore(&ms_info.mi_eplock, sflags);
}
/*
* micscif_rma_get_task:
*
* Store the parent task struct and bump up the number of remote mappings.
* If this is the first remote memory mapping for this endpoint then
* create a new rma_task_info entry in the epd task list.
*/
int micscif_rma_get_task(struct endpt *ep, int nr_pages)
{
struct list_head *item;
struct rma_task_info *info;
int err = 0;
spin_lock(&ep->lock);
list_for_each(item, &ep->rma_info.task_list) {
info = list_entry(item, struct rma_task_info, list_member);
if (info->pid == task_tgid(current)) {
info->ref_count += nr_pages;
pr_debug("%s ep %p existing pid %p ref %d\n",
__func__, ep, info->pid, info->ref_count);
goto unlock;
}
}
spin_unlock(&ep->lock);
/* A new task is mapping this window. Create a new entry */
if (!(info = kzalloc(sizeof(*info), GFP_KERNEL))) {
err = -ENOMEM;
goto done;
}
info->pid = get_pid(task_tgid(current));
info->ref_count = nr_pages;
pr_debug("%s ep %p new pid %p ref %d\n",
__func__, ep, info->pid, info->ref_count);
spin_lock(&ep->lock);
list_add_tail(&info->list_member, &ep->rma_info.task_list);
unlock:
spin_unlock(&ep->lock);
done:
return err;
}
/*
* micscif_rma_put_task:
*
* Bump down the number of remote mappings. if the ref count for this
* particular task drops to zero then remove the rma_task_info from
* the epd task list.
*/
void micscif_rma_put_task(struct endpt *ep, int nr_pages)
{
struct list_head *item;
struct rma_task_info *info;
spin_lock(&ep->lock);
list_for_each(item, &ep->rma_info.task_list) {
info = list_entry(item, struct rma_task_info, list_member);
if (info->pid == task_tgid(current)) {
info->ref_count -= nr_pages;
pr_debug("%s ep %p pid %p ref %d\n",
__func__, ep, info->pid, info->ref_count);
if (!info->ref_count) {
list_del(&info->list_member);
put_pid(info->pid);
kfree(info);
}
goto done;
}
}
/* Why was the task not found? This is a bug. */
WARN_ON(1);
done:
spin_unlock(&ep->lock);
return;
}
/* Only debug API's below */
void micscif_display_window(struct reg_range_t *window, const char *s, int line)
{
int j;
printk("%s %d window %p type %d temp %d offset 0x%llx"
" nr_pages 0x%llx nr_contig_chunks 0x%llx"
" prot %d ref_count %d magic 0x%llx peer_window 0x%llx"
" unreg_state 0x%x va_for_temp %p\n",
s, line, window, window->type, window->temp,
window->offset, window->nr_pages, window->nr_contig_chunks,
window->prot, window->ref_count, window->magic,
window->peer_window, window->unreg_state, window->va_for_temp);
for (j = 0; j < window->nr_contig_chunks; j++)
pr_debug("page[%d] = dma_addr 0x%llx num_pages 0x%x\n",
j,
window->dma_addr[j],
window->num_pages[j]);
if (RMA_WINDOW_SELF == window->type && window->pinned_pages)
for (j = 0; j < window->nr_pages; j++)
pr_debug("page[%d] = pinned_pages %p address %p\n",
j, window->pinned_pages->pages[j],
page_address(window->pinned_pages->pages[j]));
#ifdef CONFIG_ML1OM
if (window->temp_phys_addr)
for (j = 0; j < window->nr_contig_chunks; j++)
pr_debug("page[%d] = temp_phys_addr 0x%llx\n",
j, window->temp_phys_addr[j]);
if (window->phys_addr)
for (j = 0; j < window->nr_pages; j++)
pr_debug("page[%d] = phys_addr 0x%llx\n",
j, window->phys_addr[j]);
#endif
RMA_MAGIC(window);
}
Port of Intel kernel module included with MPSS 3.8.6 to newer Linux kernels.