projects / xeon-phi-kernel-module / blob
? search:
summary | tags | clone url | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Initial commit of files contained in `mpss-modules-3.8.6.tar.bz2` for Intel Xeon...
[xeon-phi-kernel-module] / include / mic / micscif.h
/*
* Copyright 2010-2017 Intel Corporation.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License, version 2,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* Disclaimer: The codes contained in these modules may be specific to
* the Intel Software Development Platform codenamed Knights Ferry,
* and the Intel product codenamed Knights Corner, and are not backward
* compatible with other Intel products. Additionally, Intel will NOT
* support the codes or instruction set in future products.
*
* Intel offers no warranty of any kind regarding the code. This code is
* licensed on an "AS IS" basis and Intel is not obligated to provide
* any support, assistance, installation, training, or other services
* of any kind. Intel is also not obligated to provide any updates,
* enhancements or extensions. Intel specifically disclaims any warranty
* of merchantability, non-infringement, fitness for any particular
* purpose, and any other warranty.
*
* Further, Intel disclaims all liability of any kind, including but
* not limited to liability for infringement of any proprietary rights,
* relating to the use of the code, even if Intel is notified of the
* possibility of such liability. Except as expressly stated in an Intel
* license agreement provided with this code and agreed upon with Intel,
* no license, express or implied, by estoppel or otherwise, to any
* intellectual property rights is granted herein.
*/
#ifndef MICSCIF_H
#define MICSCIF_H
#include <linux/errno.h>
#include <linux/hardirq.h>
#include <linux/types.h>
#include <linux/capability.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/gfp.h>
#include <linux/vmalloc.h>
#include <asm/io.h>
#include <linux/kernel.h>
#include <linux/mm_types.h>
#include <linux/jiffies.h>
#include <linux/timer.h>
#include <linux/irqflags.h>
#include <linux/time.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/semaphore.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/delay.h>
#include <linux/wait.h>
#include <asm/bug.h>
#include <linux/pci.h>
#include <linux/device.h>
#include <linux/fs.h>
#include <linux/list.h>
#include <linux/workqueue.h>
#include <linux/interrupt.h>
#include <asm/atomic.h>
#include <linux/netdevice.h>
#include <linux/debugfs.h>
#ifdef _MODULE_SCIF_
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <asm/uaccess.h>
#include <linux/poll.h>
#include <linux/mmzone.h>
#include <linux/version.h>
#endif /* MODULE_SCIF */
#include <linux/notifier.h>
#include "scif.h"
#include "mic/micbaseaddressdefine.h"
#include "mic/micsboxdefine.h"
/* The test runs in a separate thread context from the bottom
* half that processes messages from the card and setup p2p
* when these run concurrently, p2p messages get lost since they
* may be consumed by the test thread
*/
//#define ENABLE_TEST // Used to enable testing at board connect
#ifdef MIC_IS_EMULATION
#define TEST_LOOP 2
#else
#define TEST_LOOP 2000
#endif
//#define P2P_HACK 0
#include "scif.h"
#include "scif_ioctl.h"
#define SCIF_READY_MAGIC_NUM 0x1eedfee0
#ifndef SCIF_MAJOR
#define SCIF_MAJOR 0 /* Use dynamic major number by default */
#endif
#define SCIF_HOST_NODE 0 // By default the host is always node zero
#define SCIF_RMA_TEMP_CACHE_LIMIT 0x20000
/*
* The overhead for proxying a P2P DMA read to convert it to
* a DMA write by sending a SCIF Node QP message has been
* seen to be higher than programming a P2P DMA Read on self
* for transfer sizes less than the PROXY_DMA_THRESHOLD.
* The minimum threshold is different for Jaketown versus
* Ivytown and tuned for best DMA performance.
*/
#define SCIF_PROXY_DMA_THRESHOLD_JKT (32 * 1024ULL)
#define SCIF_PROXY_DMA_THRESHOLD_IVT (1024 * 1024ULL)
//#define RMA_DEBUG 0
/* Pre-defined L1_CACHE_SHIFT is 6 on RH and 7 on Suse */
#undef L1_CACHE_SHIFT
#define L1_CACHE_SHIFT 6
#undef L1_CACHE_BYTES
#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
#define MI_EPLOCK_HELD (true)
#define MAX_RDMASR 8
// Device wide SCIF information
struct micscif_info {
uint32_t mi_nodeid; // Node ID this node is to others.
struct mutex mi_conflock; // Configuration lock (used in p2p setup)
uint32_t mi_maxid; // Max known board ID
uint32_t mi_total; // Total number of running interfaces
uint32_t mi_nr_zombies; // Keep track of the number of zombie EP.
unsigned long mi_mask; // bit mask of online scif interfaces
uint64_t mi_nr_ioremap; // Keep track of number of ioremap() calls on the host
// to decide when to purge aliases for performance.
spinlock_t mi_eplock;
spinlock_t mi_connlock;
spinlock_t mi_rmalock; // Synchronize access to list of temporary registered
// windows to be destroyed.
struct mutex mi_fencelock; // Synchronize access to list of remote fences requested.
struct mutex mi_event_cblock;
spinlock_t mi_nb_connect_lock;
struct list_head mi_uaccept; // List of user acceptreq waiting for acceptreg
struct list_head mi_listen; // List of listening end points
struct list_head mi_zombie; // List of zombie end points with pending RMA's.
struct list_head mi_connected; // List of end points in connected state
struct list_head mi_disconnected; // List of end points in disconnected state
struct list_head mi_rma; // List of temporary registered windows to be destroyed.
struct list_head mi_rma_tc; // List of temporary
// registered & cached windows
// to be destroyed.
struct list_head mi_fence; // List of remote fence requests.
struct list_head mi_event_cb; /* List of event handlers registered */
struct list_head mi_nb_connect_list;
#ifdef CONFIG_MMU_NOTIFIER
struct list_head mi_mmu_notif_cleanup;
#endif
struct notifier_block mi_panic_nb;
#ifndef _MIC_SCIF_
/* The host needs to keep track of node dependencies in form of graph.
* This will need to be dynamically grown to support hotplug.
*/
uint32_t **mi_depmtrx;
/*
* Wait queue used for blocking while waiting for nodes
* to respond for disconnect message sent from host.
*/
wait_queue_head_t mi_disconn_wq;
/* stus of node remove operation*/
uint64_t mi_disconnect_status;
atomic_long_t mi_unique_msgid;
#endif
/*
* Watchdog timeout on the host. Timer expiry will result in the host
* treating the remote node as a lost node. Default value is
* DEFAULT_WATCHDOG_TO and can be modified to a value greater than 1
* second via SCIF sysfs watchdog_to entry.
*/
int mi_watchdog_to; // Watchdog timeout
int mi_watchdog_enabled; // Watchdog timeout enabled
int mi_watchdog_auto_reboot; // Watchdog auto reboot enabled
struct workqueue_struct *mi_misc_wq; // Workqueue for miscellaneous SCIF tasks.
struct work_struct mi_misc_work;
#ifdef CONFIG_MMU_NOTIFIER
struct workqueue_struct *mi_mmu_notif_wq; // Workqueue for MMU notifier cleanup tasks.
struct work_struct mi_mmu_notif_work;
#endif
int nr_gtt_entries; // GTT Debug Counter to detect leaks
uint64_t nr_2mb_pages; // Debug Counter for number of 2mb pages.
uint64_t nr_4k_pages; // Debug Counter for number of 4K pages
uint8_t en_msg_log;
wait_queue_head_t mi_exitwq;
unsigned long mi_rma_tc_limit;
uint64_t mi_proxy_dma_threshold;
#ifdef RMA_DEBUG
atomic_long_t rma_mm_cnt;
atomic_long_t rma_unaligned_cpu_cnt;
atomic_long_t rma_alloc_cnt;
atomic_long_t rma_pin_cnt;
#ifdef CONFIG_MMU_NOTIFIER
atomic_long_t mmu_notif_cnt;
#endif
#endif
#ifdef _MIC_SCIF_
int mi_intr_rcnt[MAX_RDMASR]; // Ref count to track SCIF Interrupt Handlers
#endif
struct workqueue_struct *mi_conn_wq;
struct work_struct mi_conn_work;
};
extern struct micscif_info ms_info;
#define SCIF_NODE_MAGIC_BIT 63
/* Magic value used to indicate a remote idle node without grabbing any locks */
#define SCIF_NODE_IDLE (1ULL << SCIF_NODE_MAGIC_BIT)
enum scif_state {
SCIFDEV_NOTPRESENT,
SCIFDEV_INIT,
SCIFDEV_RUNNING,
SCIFDEV_SLEEPING,
SCIFDEV_STOPPING,
SCIFDEV_STOPPED
};
extern bool mic_p2p_enable;
extern bool mic_p2p_proxy_enable;
extern bool mic_reg_cache_enable;
extern bool mic_ulimit_check;
/* p2p mapping from node id to peer id */
struct scif_p2p_info {
int ppi_peer_id;
struct scatterlist *ppi_sg[2];
uint64_t sg_nentries[2]; // no of entries in scatterlists
dma_addr_t ppi_pa[2]; // one for mmio; one for aper
dma_addr_t ppi_mic_addr[2]; // one for mmio; one for aper
uint64_t ppi_len[2];
#define PPI_MMIO 0
#define PPI_APER 1
enum scif_state ppi_disc_state; //Disconnection state of this peer node.
struct list_head ppi_list;
};
/* one per remote node */
struct micscif_dev {
uint16_t sd_node;
enum scif_state sd_state;
volatile void *mm_sbox;
uint64_t sd_base_addr; /* Remote node's base bus addr
* for the local node's aperture
*/
#ifndef _MIC_SCIF_
struct list_head sd_p2p; /* List of bus addresses for
* other nodes, these are allocated
* by the host driver and are
* valid only on the host node
*/
struct delayed_work sd_watchdog_work;
wait_queue_head_t sd_watchdog_wq;
struct workqueue_struct *sd_ln_wq;
char sd_ln_wqname[16];
#endif
int n_qpairs; /* FIXME:
* This is always set to 1,
*/
struct micscif_qp *qpairs; /* Same FIXME as above
* There is single qp established
* with this remote node
*/
struct workqueue_struct *sd_intr_wq; /* sd_intr_wq & sd_intr_bh
* together constitute the workqueue
* infrastructure needed to
* run the bottom half handler
* for messages received from
* this node
*/
char sd_intr_wqname[16];
struct work_struct sd_intr_bh;
unsigned int sd_intr_handle;
uint32_t sd_rdmasr;
struct workqueue_struct *sd_loopb_wq;
char sd_loopb_wqname[16];
struct work_struct sd_loopb_work;
struct list_head sd_loopb_recv_q;
/* Lock to synchronize remote node state transitions */
struct mutex sd_lock;
/*
* Global Ref count per SCIF device tracking all SCIF API's which
* might communicate across PCIe.
*/
atomic_long_t scif_ref_cnt;
/*
* Global Ref count per SCIF device tracking scif_mmap()/
* scif_get_pages(). sd_lock protects scif_map_ref_cnt
* hence it does not need to be an atomic operation. Note that
* scif_mmap()/scif_get_pages() is not in the critical
* perf path.
*/
int scif_map_ref_cnt;
/*
* Wait queue used for blocking while waiting for nodes
* to wake up or to be removed.
*/
wait_queue_head_t sd_wq;
uint64_t sd_wait_status;
#ifdef _MIC_SCIF_
wait_queue_head_t sd_p2p_wq;
bool sd_proxy_dma_reads;
struct delayed_work sd_p2p_dwork;
int sd_p2p_retry;
#endif
/*
* The NUMA node the peer is attached to on the host.
*/
int sd_numa_node;
/*
* Waitqueue for blocking while waiting for remote memory
* mappings to drop to zero.
*/
wait_queue_head_t sd_mmap_wq;
/* When a nodeqp message is received, this is set.
* And it is reset by the watchdog time */
atomic_t sd_node_alive;
int num_active_conn;
#ifdef ENABLE_TEST
struct workqueue_struct *producer;
struct workqueue_struct *consumer;
char producer_name[16];
char consumer_name[16];
struct work_struct producer_work;
struct work_struct consumer_work;
int count;
int test_done;
#endif // ENABLE_TEST
};
extern struct micscif_dev scif_dev[];
#include "mic/micscif_nodeqp.h"
#include "mic/micscif_nm.h"
#include "mic/micscif_smpt.h"
#include "mic/micscif_va_gen.h"
#include "mic/mic_dma_api.h"
#include "mic/mic_dma_lib.h"
#include "mic/micscif_rma.h"
#include "mic/micscif_rma_list.h"
/*
* data structure used to sync SCIF_GET_NODE_INFO messaging
*/
struct get_node_info {
enum micscif_msg_state state;
wait_queue_head_t wq;
};
static inline uint64_t align_low(uint64_t data, uint32_t granularity)
{
return ALIGN(data - (granularity - 1), granularity);
}
#define SCIF_MIN(a, b) (((a) < (b)) ? (a) : (b))
#define SCIF_MAX(a, b) (((a) > (b)) ? (a) : (b))
enum endptstate {
SCIFEP_CLOSED, // Internal state
SCIFEP_UNBOUND, // External state
SCIFEP_BOUND, // External state
SCIFEP_LISTENING, // External state
SCIFEP_CONNECTED, // External state
SCIFEP_CONNECTING, // Internal state
SCIFEP_MAPPING, // Internal state
SCIFEP_CLOSING, // Internal state
SCIFEP_CLLISTEN, // Internal state
SCIFEP_DISCONNECTED, // Internal state
SCIFEP_ZOMBIE // Internal state
};
extern char *scif_ep_states[];
// Used for coordinating connection accept sequence. This is the data structure
// for the conlist in the endpoint.
struct conreq {
struct nodemsg msg;
struct list_head list;
};
/* Size of the RB for the Node QP */
#define NODE_QP_SIZE 0x10000
/* Size of the RB for the Endpoint QP */
#define ENDPT_QP_SIZE 0x1000
struct endpt_qp_info {
/* Qpair for this endpoint */
struct micscif_qp *qp;
/*
* Physical addr of the QP for Host or
* GTT offset of the QP for MIC.
* Required for unmapping the QP during close.
*/
dma_addr_t qp_offset;
/*
* Payload in a SCIF_CNCT_GNT message containing the
* physical address of the remote_qp.
*/
dma_addr_t cnct_gnt_payload;
};
#define SCIFEP_MAGIC 0x5c1f000000005c1f
struct endpt {
volatile enum endptstate state;
spinlock_t lock;
struct scif_portID port;
struct scif_portID peer;
int backlog;
struct endpt_qp_info qp_info;
struct endpt_rma_info rma_info;
/*
* scifdev used by this endpt to communicate with remote node.
*/
struct micscif_dev *remote_dev;
uint64_t remote_ep;
/*
* Keep track of number of connection requests.
*/
int conreqcnt;
/*
* Cache remote SCIF device state.
*/
enum scif_state sd_state;
/*
* True if the endpoint was created
* via scif_accept(..).
*/
bool accepted_ep;
/*
* Open file information used to match the id passed
* in with the flush routine.
*/
struct files_struct *files;
/*
* Reference count for functions using this endpoint.
*/
struct kref ref_count;
struct list_head conlist;
wait_queue_head_t conwq;
wait_queue_head_t disconwq;
wait_queue_head_t diswq;
wait_queue_head_t sendwq;
wait_queue_head_t recvwq;
struct mutex sendlock;
struct mutex recvlock;
struct list_head list;
#ifdef CONFIG_MMU_NOTIFIER
struct list_head mmu_list;
#endif
struct list_head li_accept; /* pending ACCEPTREG */
int acceptcnt; /* pending ACCEPTREG cnt */
struct list_head liacceptlist; /* link to listen accept */
struct list_head miacceptlist; /* link to mi_uaccept */
struct endpt *listenep; /* associated listen ep */
/* Non-blocking connect */
struct work_struct conn_work;
struct scif_portID conn_port;
int conn_err;
int conn_async_state;
wait_queue_head_t conn_pend_wq;
struct list_head conn_list;
};
static __always_inline void
micscif_queue_for_cleanup(struct reg_range_t *window, struct list_head *list)
{
struct endpt *ep = (struct endpt *)window->ep;
INIT_LIST_HEAD(&window->list_member);
window->dma_mark = get_dma_mark(ep->rma_info.dma_chan);
spin_lock(&ms_info.mi_rmalock);
list_add_tail(&window->list_member, list);
spin_unlock(&ms_info.mi_rmalock);
queue_work(ms_info.mi_misc_wq, &ms_info.mi_misc_work);
}
static __always_inline void
__micscif_rma_destroy_tcw_helper(struct reg_range_t *window)
{
list_del(&window->list_member);
micscif_queue_for_cleanup(window, &ms_info.mi_rma_tc);
}
void print_ep_state(struct endpt *ep, char *label);
// Function prototypes needed by Unix/Linux drivers linking to scif
int scif_fdopen(struct file *f);
int scif_fdclose(struct file *f);
int scif_process_ioctl(struct file *f, unsigned int cmd, uint64_t arg);
int micscif_mmap(struct file *file, struct vm_area_struct *vma);
int scif_mmap(struct vm_area_struct *vma, scif_epd_t epd);
void scif_munmap(struct vm_area_struct *vma);
void scif_proc_init(void);
void scif_proc_cleanup(void);
int scif_user_send(scif_epd_t epd, void *msg, int len, int flags);
int scif_user_recv(scif_epd_t epd, void *msg, int len, int flags);
int __scif_pin_pages(void *addr, size_t len, int *out_prot,
int map_flags, scif_pinned_pages_t *pages);
scif_epd_t __scif_open(void);
int __scif_bind(scif_epd_t epd, uint16_t pn);
int __scif_listen(scif_epd_t epd, int backlog);
int __scif_connect(scif_epd_t epd, struct scif_portID *dst, bool non_block);
int __scif_accept(scif_epd_t epd, struct scif_portID *peer, scif_epd_t
*newepd, int flags);
int __scif_close(scif_epd_t epd);
int __scif_send(scif_epd_t epd, void *msg, int len, int flags);
int __scif_recv(scif_epd_t epd, void *msg, int len, int flags);
off_t __scif_register(scif_epd_t epd, void *addr, size_t len, off_t offset,
int prot_flags, int map_flags);
int __scif_unregister(scif_epd_t epd, off_t offset, size_t len);
int __scif_readfrom(scif_epd_t epd, off_t loffset, size_t len, off_t
roffset, int rma_flags);
int __scif_writeto(scif_epd_t epd, off_t loffset, size_t len, off_t
roffset, int rma_flags);
int __scif_fence_mark(scif_epd_t epd, int flags, int *mark);
int __scif_fence_wait(scif_epd_t epd, int mark);
int __scif_fence_signal(scif_epd_t epd, off_t loff, uint64_t lval, off_t roff,
uint64_t rval, int flags);
off_t __scif_register_pinned_pages(scif_epd_t epd,
scif_pinned_pages_t pinned_pages, off_t offset, int map_flags);
int __scif_get_pages(scif_epd_t epd, off_t offset, size_t len,
struct scif_range **pages);
int __scif_put_pages(struct scif_range *pages);
int __scif_flush(scif_epd_t epd);
void micscif_misc_handler(struct work_struct *work);
void micscif_conn_handler(struct work_struct *work);
uint16_t rsrv_scif_port(uint16_t port);
uint16_t get_scif_port(void);
void put_scif_port(uint16_t port);
void micscif_send_exit(void);
void scif_ref_rel(struct kref *kref_count);
#ifdef _MODULE_SCIF_
unsigned int micscif_poll(struct file *f, poll_table *wait);
unsigned int scif_pollfd(struct file *f, poll_table *wait, scif_epd_t epd);
unsigned int __scif_pollfd(struct file *f, poll_table *wait, struct endpt *ep);
int micscif_flush(struct file *f, fl_owner_t id);
#endif
#ifdef _MIC_SCIF_
void mic_debug_init(void);
void micscif_get_node_info(void);
void scif_poll_qp_state(struct work_struct *work);
#endif
void mic_debug_uninit(void);
#define serializing_request(x) ((void)*(volatile uint8_t*)(x))
// State list helper functions.
// Each of these functions must be called with the end point lock unlocked. If
// the end point is found on the list the end point returned will have its lock
// set and sflags will return the value to be used to do an unlock_irqrestore
// at the end of the calling function.
static inline struct endpt *
micscif_find_listen_ep(uint16_t port, unsigned long *sflags)
{
struct endpt *ep = NULL;
struct list_head *pos, *tmpq;
unsigned long flags;
spin_lock_irqsave(&ms_info.mi_eplock, flags);
list_for_each_safe(pos, tmpq, &ms_info.mi_listen) {
ep = list_entry(pos, struct endpt, list);
if (ep->port.port == port) {
*sflags = flags;
spin_lock(&ep->lock);
spin_unlock(&ms_info.mi_eplock);
return ep;
}
}
spin_unlock_irqrestore(&ms_info.mi_eplock, flags);
return (struct endpt *)NULL;
}
// Must be called with end point locked
static inline struct conreq *
miscscif_get_connection_request(struct endpt *ep, uint64_t payload)
{
struct conreq *conreq;
struct list_head *pos, *tmpq;
list_for_each_safe(pos, tmpq, &ep->conlist) {
conreq = list_entry(pos, struct conreq, list);
if (conreq->msg.payload[0] == payload) {
list_del(pos);
ep->conreqcnt--;
return conreq;
}
}
return (struct conreq *)NULL;
}
// There is no requirement for the callee to have the end point
// locked like other API's above.
static inline void
micscif_remove_zombie_ep(struct endpt *ep)
{
struct list_head *pos, *tmpq;
unsigned long sflags;
struct endpt *tmpep;
spin_lock_irqsave(&ms_info.mi_eplock, sflags);
list_for_each_safe(pos, tmpq, &ms_info.mi_zombie) {
tmpep = list_entry(pos, struct endpt, list);
if (tmpep == ep) {
list_del(pos);
ms_info.mi_nr_zombies--;
}
}
spin_unlock_irqrestore(&ms_info.mi_eplock, sflags);
}
static inline void
micscif_cleanup_zombie_epd(void)
{
struct list_head *pos, *tmpq;
unsigned long sflags;
struct endpt *ep;
spin_lock_irqsave(&ms_info.mi_eplock, sflags);
list_for_each_safe(pos, tmpq, &ms_info.mi_zombie) {
ep = list_entry(pos, struct endpt, list);
if (micscif_rma_ep_can_uninit(ep)) {
list_del(pos);
ms_info.mi_nr_zombies--;
va_gen_destroy(&ep->rma_info.va_gen);
kfree(ep);
}
}
spin_unlock_irqrestore(&ms_info.mi_eplock, sflags);
}
#define SCIF_WAKE_UP_SEND (1 << 1)
#define SCIF_WAKE_UP_RECV (1 << 2)
/**
* scif_wakeup_ep() - Wake up all clients based on the type
* requested i.e. threads blocked in scif_send(..) and/or scif_recv(..).
*/
static inline void
scif_wakeup_ep(int type)
{
struct endpt *ep;
unsigned long sflags;
struct list_head *pos, *tmpq;
spin_lock_irqsave(&ms_info.mi_connlock, sflags);
list_for_each_safe(pos, tmpq, &ms_info.mi_connected) {
ep = list_entry(pos, struct endpt, list);
if (type & SCIF_WAKE_UP_SEND)
wake_up_interruptible(&ep->sendwq);
if (type & SCIF_WAKE_UP_RECV)
wake_up_interruptible(&ep->recvwq);
}
spin_unlock_irqrestore(&ms_info.mi_connlock, sflags);
}
/*
* is_self_scifdev:
* @dev: The remote SCIF Device
*
* Returns true if the SCIF Device passed is the self aka Loopback SCIF device.
*/
static inline int is_self_scifdev(struct micscif_dev *dev)
{
return dev->sd_node == ms_info.mi_nodeid;
}
/*
* is_p2p_scifdev:
* @dev: The remote SCIF Device
*
* Returns true if the SCIF Device is a MIC Peer to Peer SCIF device.
*/
static inline bool is_p2p_scifdev(struct micscif_dev *dev)
{
#ifdef _MIC_SCIF_
return dev != &scif_dev[SCIF_HOST_NODE] && !is_self_scifdev(dev);
#else
return false;
#endif
}
/*
* get_conn_count:
* @dev: The remote SCIF Device
*
* Increments the number of active SCIF connections. Callee is expected
* to synchronize calling this API with put_conn_count.
*/
static __always_inline void
get_conn_count(struct micscif_dev *dev)
{
dev->num_active_conn++;
}
/*
* put_conn_count:
* @dev: The remote SCIF Device
*
* Decrements the number of active connections. Callee is expected
* to synchronize calling this API with get_conn_count.
*/
static __always_inline void
put_conn_count(struct micscif_dev *dev)
{
dev->num_active_conn--;
BUG_ON(dev->num_active_conn < 0);
}
/*
* get_kref_count:
* epd: SCIF endpoint
*
* Increments kmod endpoint reference count. Callee is expected
* to synchronize calling this API with put_kref_count.
*/
static __always_inline void
get_kref_count(scif_epd_t epd)
{
kref_get(&(epd->ref_count));
}
/*
* put_kref_count:
* epd: SCIF endpoint
*
* Decrements kmod endpoint reference count. Callee is expected
* to synchronize calling this API with get_kref_count.
*/
static __always_inline void
put_kref_count(scif_epd_t epd)
{
kref_put(&(epd->ref_count), scif_ref_rel);
}
/*
* is_scifdev_alive:
* @dev: The remote SCIF Device
*
* Returns true if the remote SCIF Device is running or sleeping for
* this endpoint.
*/
static inline int scifdev_alive(struct endpt *ep)
{
return (((SCIFDEV_RUNNING == ep->remote_dev->sd_state) ||
(SCIFDEV_SLEEPING == ep->remote_dev->sd_state)) &&
SCIFDEV_RUNNING == ep->sd_state);
}
/*
* verify_epd:
* ep: SCIF endpoint
*
* Checks several generic error conditions and returns the
* appropiate error.
*/
static inline int verify_epd(struct endpt *ep)
{
if (ep->state == SCIFEP_DISCONNECTED)
return -ECONNRESET;
if (ep->state != SCIFEP_CONNECTED)
return -ENOTCONN;
if (!scifdev_alive(ep))
return -ENODEV;
return 0;
}
/**
* scif_invalidate_ep() - Set remote SCIF device state for all connected
* and disconnected endpoints for a particular node to SCIFDEV_STOPPED,
* change endpoint state to disconnected and wake up all send/recv/con
* waitqueues.
*/
static inline void
scif_invalidate_ep(int node)
{
struct endpt *ep;
unsigned long sflags;
struct list_head *pos, *tmpq;
spin_lock_irqsave(&ms_info.mi_connlock, sflags);
list_for_each_safe(pos, tmpq, &ms_info.mi_disconnected) {
ep = list_entry(pos, struct endpt, list);
if (ep->remote_dev->sd_node == node) {
spin_lock(&ep->lock);
ep->sd_state = SCIFDEV_STOPPED;
spin_unlock(&ep->lock);
}
}
list_for_each_safe(pos, tmpq, &ms_info.mi_connected) {
ep = list_entry(pos, struct endpt, list);
if (ep->remote_dev->sd_node == node) {
list_del(pos);
put_conn_count(ep->remote_dev);
spin_lock(&ep->lock);
ep->state = SCIFEP_DISCONNECTED;
list_add_tail(&ep->list, &ms_info.mi_disconnected);
ep->sd_state = SCIFDEV_STOPPED;
wake_up_interruptible(&ep->sendwq);
wake_up_interruptible(&ep->recvwq);
wake_up_interruptible(&ep->conwq);
spin_unlock(&ep->lock);
}
}
spin_unlock_irqrestore(&ms_info.mi_connlock, sflags);
flush_workqueue(ms_info.mi_conn_wq);
}
/*
* Only Debug Functions Below
*/
#define SCIF_CRUMB pr_debug("%s %d\n", __func__, __LINE__)
static inline void
micscif_display_all_zombie_ep(void)
{
struct list_head *pos, *tmpq;
unsigned long sflags;
struct endpt *ep;
pr_debug("Zombie Info Start\n");
spin_lock_irqsave(&ms_info.mi_eplock, sflags);
list_for_each_safe(pos, tmpq, &ms_info.mi_zombie) {
ep = list_entry(pos, struct endpt, list);
if (!list_empty(&ep->rma_info.reg_list))
micscif_display_all_windows(&ep->rma_info.reg_list);
if (!list_empty(&ep->rma_info.remote_reg_list))
micscif_display_all_windows(
&ep->rma_info.remote_reg_list);
}
spin_unlock_irqrestore(&ms_info.mi_eplock, sflags);
pr_debug("Zombie Info End\n");
}
static inline void dump_ep(scif_epd_t epd, const char *func, int line)
{
struct endpt *ep = (struct endpt *)epd;
pr_debug("%s %d state %d lock %p port.node 0x%x"
"port.port 0x%x peer.node 0x%x peer.port 0x%x backlog %d qp %p"
"qp_offset 0x%llx cnct_gnt_payload 0x%llx remote_dev %p\n",
func, line, ep->state, &ep->lock, ep->port.node,
ep->port.port, ep->peer.node, ep->peer.port, ep->backlog,
ep->qp_info.qp, ep->qp_info.qp_offset,
ep->qp_info.cnct_gnt_payload, ep->remote_dev);
}
static inline void dump_qp(volatile struct micscif_qp *qp, const char *func, int line)
{
pr_debug("%s %d qp %p local_buf 0x%llx"
" local_qp 0x%llx remote_buf 0x%llx remote_qp %p ep 0x%llx\n",
func, line, qp, qp->local_buf,
qp->local_qp, qp->remote_buf, qp->remote_qp, qp->ep);
}
static inline void dump_rb(struct micscif_rb *rb, const char *func, int line)
{
pr_debug("%s %d rb %p rb_base %p *read_ptr 0x%x"
" *write_ptr 0x%x size 0x%x"
" cro 0x%x cwo 0x%x ocro 0x%x ocwo 0x%x\n",
func, line, rb, rb->rb_base, *rb->read_ptr,
*rb->write_ptr, rb->size, rb->current_read_offset,
rb->current_write_offset,
rb->old_current_read_offset,
rb->old_current_write_offset);
}
#endif /* MICSCIF_H */
Port of Intel kernel module included with MPSS 3.8.6 to newer Linux kernels.