[xeon-phi-kernel-module] / dma / mic_dma_lib.c

/*
 * Copyright 2010-2017 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 *
 * Disclaimer: The codes contained in these modules may be specific to
 * the Intel Software Development Platform codenamed Knights Ferry,
 * and the Intel product codenamed Knights Corner, and are not backward
 * compatible with other Intel products. Additionally, Intel will NOT
 * support the codes or instruction set in future products.
 *
 * Intel offers no warranty of any kind regarding the code. This code is
 * licensed on an "AS IS" basis and Intel is not obligated to provide
 * any support, assistance, installation, training, or other services
 * of any kind. Intel is also not obligated to provide any updates,
 * enhancements or extensions. Intel specifically disclaims any warranty
 * of merchantability, non-infringement, fitness for any particular
 * purpose, and any other warranty.
 *
 * Further, Intel disclaims all liability of any kind, including but
 * not limited to liability for infringement of any proprietary rights,
 * relating to the use of the code, even if Intel is notified of the
 * possibility of such liability. Except as expressly stated in an Intel
 * license agreement provided with this code and agreed upon with Intel,
 * no license, express or implied, by estoppel or otherwise, to any
 * intellectual property rights is granted herein.
 */

#include<linux/module.h>
#include<linux/init.h>
#include<linux/slab.h>
#include<asm/io.h>
#include<linux/mm.h>
#include<linux/kernel.h>
#include<linux/interrupt.h>
#include<linux/proc_fs.h>
#include<linux/bitops.h>
#include<linux/version.h>
#ifdef _MIC_SCIF_
#include <asm/mic/mic_common.h>
#ifdef CONFIG_PAGE_CACHE_DMA
#include <linux/mic_dma/mic_dma_callback.h>
#endif
#endif

#ifndef _MIC_SCIF_
#include <mic/micscif.h>
#include "mic_common.h"
#endif

#include <mic/mic_dma_lib.h>
#include <mic/micscif_smpt.h>
#include <mic/mic_dma_md.h>
#include <mic/mic_dma_api.h>
#include <mic/compl_buf_ring.h>
#include <mic/micscif_smpt.h>
#include <mic/micsboxdefine.h>

MODULE_LICENSE("GPL");

#ifdef MIC_IS_EMULATION
#define DMA_TO		(INT_MAX)
#define DMA_FENCE_TIMEOUT_CNT (INT_MAX)
#else
#define DMA_TO		(5 * HZ)
#define DMA_SLOWEST_BW  (300)  // 300Mbps
// the maximum size for each decriptor entry is 2M
#define DMA_FENCE_TIMEOUT_CNT (2 * MIC_MAX_NUM_DESC_PER_RING /DMA_SLOWEST_BW/ (DMA_TO/HZ))
#endif

#ifdef _MIC_SCIF_
#define MAX_DMA_XFER_SIZE  MIC_MAX_DMA_XFER_SIZE
#else
/* Use 512K as the maximum descriptor transfer size for Host */
#define MAX_DMA_XFER_SIZE  (((1U) * 1024 * 1024) >> 1)
#endif
#ifndef KASSERT
#define KASSERT(x, y, ...)		\
	do {				\
		if(!x)			\
			printk(y, ##__VA_ARGS__);\
		BUG_ON(!x);		\
	} while(0)
#endif
/*
 * Arrary of per device DMA contexts. The card only uses index 0. The host uses one
 * context per card starting from 0.
 */
static struct mic_dma_ctx_t *mic_dma_context[MAX_BOARD_SUPPORTED + 1];
static struct mutex lock_dma_dev_init[MAX_BOARD_SUPPORTED + 1];

enum mic_desc_format_type {
	NOP,
	MEMCOPY,
	STATUS,
	GENERAL,
	KEYNONCECNT,
	KEY
};
char proc_dma_reg[]="mic_dma_registers_";
char proc_dma_ring[]="mic_dma_ring_";

#define PR_PREFIX "DMA_LIB_MI:"
#define DMA_DESC_RING_SIZE MIC_MAX_NUM_DESC_PER_RING
#define MAX_POLLING_BUFFERS DMA_DESC_RING_SIZE

#define DMA_PROC
static void mic_dma_proc_init(struct mic_dma_ctx_t *dma_ctx);
static void mic_dma_proc_uninit(struct mic_dma_ctx_t *dma_ctx);

/*
 * TODO: This is size of s/w interrupt ring.
 * We need to figure out a value so that we don't run out of memory in
 * interrupt ring and at the same time don't waste memory
 */
#define NUM_COMP_BUFS (((PAGE_SIZE/sizeof(struct dma_completion_cb*)) - 10) * 10)

struct intr_compl_buf_ring {
	struct dma_completion_cb **comp_cb_array;
	struct compl_buf_ring ring;
	int old_tail;
};

struct mic_dma_ctx_t;			/* Forward Declaration */

struct dma_channel {
	int ch_num;/*Duplicated in md_mic_dma_chan struct too*/
	struct md_mic_dma_chan *chan;
	atomic_t flags;
	wait_queue_head_t intr_wq;
	wait_queue_head_t access_wq;
	union md_mic_dma_desc *desc_ring_bak;
	union md_mic_dma_desc *desc_ring;
	phys_addr_t desc_ring_phys;
	uint64_t next_write_index; /* next write index into desc ring */
	struct intr_compl_buf_ring intr_ring;
	struct compl_buf_ring poll_ring;
	struct mic_dma_ctx_t *dma_ctx;	  /* Pointer to parent DMA context */
};

/* Per MIC device (per MIC board) DMA context */
struct mic_dma_ctx_t {
	struct dma_channel	dma_channels[MAX_NUM_DMA_CHAN];
	int			last_allocated_dma_channel_num;
	struct mic_dma_device	dma_dev;
	int			device_num;
	atomic_t		ref_count;	/* Reference count */
	atomic_t		ch_num;
};

/* DMA Library Init/Uninit Routines */
static int mic_dma_lib_init(uint8_t *mmio_va_base, struct mic_dma_ctx_t *dma_ctx);
static void mic_dma_lib_uninit(struct mic_dma_ctx_t *dma_ctx);

int get_chan_num(struct dma_channel *chan)
{
	return chan->ch_num;
}
EXPORT_SYMBOL(get_chan_num);

void initdmaglobalvar(void)
{
	memset(mic_dma_context, 0, sizeof(struct mic_dma_ctx_t *) * (MAX_BOARD_SUPPORTED + 1));
}

static void
ack_dma_interrupt(struct dma_channel *ch)
{
	md_mic_dma_chan_mask_intr(&ch->dma_ctx->dma_dev, ch->chan);
	md_mic_dma_chan_unmask_intr(&ch->dma_ctx->dma_dev, ch->chan);
}

/* Returns true if the next write index is "within" bounds */
static inline bool verify_next_write_index(struct dma_channel *ch)
{
	bool ret = false;

	if (ch->next_write_index < DMA_DESC_RING_SIZE)
		ret = true;
	else
		printk(KERN_ERR "%s %d OOB ch_num 0x%x next_write_index 0x%llx\n", 
			__func__, __LINE__, 
			ch->ch_num, ch->next_write_index);
	return ret;
}

/* TODO:
 * See if we can use __get_free_pages or something similar
 * get_free_pages expects a power of 2 number of pages
 */
static void
alloc_dma_desc_ring_mem(struct dma_channel *ch, struct mic_dma_ctx_t *dma_ctx)
{
#ifndef _MIC_SCIF_
	struct pci_dev *pdev;
#endif
	/* Is there any kernel allocator which provides the
	 * option to give the alignment??
	 */
	ch->desc_ring = kzalloc(
			(DMA_DESC_RING_SIZE * sizeof(*ch->desc_ring)) + PAGE_SIZE, GFP_KERNEL);
	ch->desc_ring_bak = ch->desc_ring;
	ch->desc_ring = (union md_mic_dma_desc *)ALIGN(
			(uint64_t)ch->desc_ring, PAGE_SIZE);
#ifdef _MIC_SCIF_
	ch->desc_ring_phys = virt_to_phys(ch->desc_ring);
#else
	micscif_pci_dev(dma_ctx->device_num, &pdev);
	ch->desc_ring_phys = mic_map_single(dma_ctx->device_num - 1, pdev, (void *)ch->desc_ring,
			(DMA_DESC_RING_SIZE * sizeof(*ch->desc_ring)) + PAGE_SIZE);
	BUG_ON(pci_dma_mapping_error(pdev, ch->desc_ring_phys));
#endif
}

/*
 * Call completion cb functions:
 * Take care of case where we allocated temp buf
 */
static void
mic_dma_lib_interrupt_handler(struct dma_channel *chan)
{
	int i = 0;
	int ring_size = chan->intr_ring.ring.size;
	struct dma_completion_cb **temp = chan->intr_ring.comp_cb_array;
	struct dma_completion_cb *cb;
	int new_tail, old_tail;

	if (mic_hw_family(chan->dma_ctx->device_num) == FAMILY_KNC &&
		mic_hw_stepping(chan->dma_ctx->device_num) >= KNC_B0_STEP) {
		unsigned long error = *((uint32_t*)chan->chan->dstat_wb_loc);
		if (unlikely(test_bit(31, &error)))
			printk(KERN_ERR "DMA h/w error - %s %d, dstatwb=%lx\n", 
					__func__, __LINE__, error);
	}
	new_tail = read_tail(&chan->intr_ring.ring);
	old_tail = chan->intr_ring.old_tail;

	for (; i < ring_size && old_tail != new_tail;
		old_tail = incr_rb_index(old_tail, ring_size), i++) {
		cb = (struct dma_completion_cb *)xchg(&temp[old_tail], NULL);
		if (cb) {
			cb->dma_completion_func(cb->cb_cookie);
		}
	}
	chan->intr_ring.old_tail = new_tail;
	update_tail(&chan->intr_ring.ring, new_tail);
	wake_up(&chan->intr_wq);
	if (i == ring_size && old_tail != new_tail) {
		printk(KERN_ERR PR_PREFIX "Something went wrong, old tail = %d, new tail = %d\n", 
							old_tail, new_tail);
	}
}

#ifdef _MIC_SCIF_
/*
 * TODO;
 * Maybe move the logic into slow interrupt handler
 */
static irqreturn_t
dma_interrupt_handler(int irq, void *dev_id)
{
	struct dma_channel *chan = ((struct dma_channel*)dev_id);

	ack_dma_interrupt(chan);
	mic_dma_lib_interrupt_handler(chan);

	return IRQ_HANDLED;
}
#else

#define SBOX_SICR0_DMA(x)  (((x) >> 8) & 0xff)

/*
 * TODO;
 * Maybe move the logic into slow interrupt handler
 */
void
host_dma_interrupt_handler(mic_dma_handle_t dma_handle, uint32_t sboxSicr0reg)
{
	struct mic_dma_ctx_t *dma_ctx = (struct mic_dma_ctx_t *) dma_handle;
	uint32_t dma_chan_id;
	struct dma_channel *ch;

	for (dma_chan_id = 0; dma_chan_id < 8; dma_chan_id++) {
		if (SBOX_SICR0_DMA(sboxSicr0reg) & (0x1 << dma_chan_id)) {
			ch = &dma_ctx->dma_channels[dma_chan_id];
			if (ch->desc_ring)
				host_dma_lib_interrupt_handler(ch);
		}
	}
}

void
host_dma_lib_interrupt_handler(struct dma_channel *chan)
{
	ack_dma_interrupt(chan);
	mic_dma_lib_interrupt_handler(chan);
}
#endif

static void
mi_mic_dma_chan_setup(struct dma_channel *ch, struct mic_dma_ctx_t *dma_ctx)
{
	ch->next_write_index = ch->chan->cached_tail;

	init_ring(&ch->poll_ring, MAX_POLLING_BUFFERS, dma_ctx->device_num);

	ch->intr_ring.comp_cb_array =
		kzalloc(sizeof(*ch->intr_ring.comp_cb_array) * NUM_COMP_BUFS, GFP_KERNEL);
	init_ring(&ch->intr_ring.ring, NUM_COMP_BUFS, dma_ctx->device_num);
	ch->intr_ring.old_tail = 0;
}

static void
mi_mic_dma_chan_destroy(struct dma_channel *ch, struct mic_dma_ctx_t *dma_ctx)
{
	uninit_ring(&ch->intr_ring.ring, dma_ctx->device_num);
	kfree(ch->intr_ring.comp_cb_array);
	uninit_ring(&ch->poll_ring, dma_ctx->device_num);
}

int
open_dma_device(int device_num, uint8_t *mmio_va_base, mic_dma_handle_t* dma_handle)
{
	int result = 0;

	if (device_num >= MAX_BOARD_SUPPORTED)
		return -EINVAL;

	mutex_lock(&lock_dma_dev_init[device_num]);
	if (!mic_dma_context[device_num]) {
		mic_dma_context[device_num] = kzalloc(sizeof(struct mic_dma_ctx_t), GFP_KERNEL);
		BUG_ON(!mic_dma_context[device_num]);

		mic_dma_context[device_num]->device_num = device_num;

		result = mic_dma_lib_init(mmio_va_base, mic_dma_context[device_num]);
		BUG_ON(result);
	}

	atomic_inc(&mic_dma_context[device_num]->ref_count);
	*dma_handle = mic_dma_context[device_num];
	mutex_unlock(&lock_dma_dev_init[device_num]);

	return result;
}
EXPORT_SYMBOL(open_dma_device);

void
close_dma_device(int device_num, mic_dma_handle_t *dma_handle)
{
	struct mic_dma_ctx_t *dma_ctx;

	if (device_num >= MAX_BOARD_SUPPORTED)
		return;

	mutex_lock(&lock_dma_dev_init[device_num]);
	dma_ctx = (struct mic_dma_ctx_t *) *dma_handle;
	if (dma_ctx &&
		atomic_read(&dma_ctx->ref_count) &&
		atomic_dec_and_test(&dma_ctx->ref_count)) {
		mic_dma_lib_uninit(dma_ctx);
		mic_dma_context[dma_ctx->device_num] = 0;
		*dma_handle = NULL;
		kfree(dma_ctx);
	}
	mutex_unlock(&lock_dma_dev_init[device_num]);
}
EXPORT_SYMBOL(close_dma_device);

void mi_mic_dma_chan_set_dstat_wb(struct mic_dma_ctx_t *dma_ctx,
				struct md_mic_dma_chan *chan)
{
#ifndef _MIC_SCIF_
	struct pci_dev *pdev;
#endif
	if (!chan->dstat_wb_phys) {
		chan->dstat_wb_loc = kzalloc(sizeof(uint32_t), GFP_KERNEL);

#ifdef _MIC_SCIF_
		chan->dstat_wb_phys = virt_to_phys(chan->dstat_wb_loc);
#else
		micscif_pci_dev(dma_ctx->device_num, &pdev);
		chan->dstat_wb_phys = mic_map_single(dma_ctx->device_num - 1, pdev, chan->dstat_wb_loc,
			sizeof(uint32_t));
		BUG_ON(pci_dma_mapping_error(pdev, chan->dstat_wb_phys));
#endif
	}
	md_mic_dma_chan_set_dstat_wb(&dma_ctx->dma_dev, chan);
}

void
md_mic_dma_chan_setup(struct mic_dma_ctx_t *dma_ctx, struct dma_channel *ch)
{
	md_mic_dma_chan_unmask_intr(&dma_ctx->dma_dev, ch->chan);

	/*
	 * Disable the channel, update desc ring base and size, write new head
	 * and then enable the channel.
	 */
	if (mic_hw_family(ch->dma_ctx->device_num) == FAMILY_KNC &&
		mic_hw_stepping(ch->dma_ctx->device_num) >= KNC_B0_STEP) {
		mi_mic_dma_chan_set_dstat_wb(dma_ctx, ch->chan);
		md_mic_dma_chan_set_dcherr_msk(&dma_ctx->dma_dev, ch->chan, 0);
	}
	md_mic_dma_chan_set_desc_ring(&dma_ctx->dma_dev, ch->chan,
			ch->desc_ring_phys,
			DMA_DESC_RING_SIZE);

	wmb();

	md_mic_dma_chan_unmask_intr(&dma_ctx->dma_dev, ch->chan);
}

int
mic_dma_lib_init(uint8_t *mmio_va_base, struct mic_dma_ctx_t *dma_ctx)
{
	int i;
#ifdef _MIC_SCIF_
	int ret_value;
#endif
	struct dma_channel *ch;
	enum md_mic_dma_chan_owner owner, currentOwner;

	//pr_debug(PR_PREFIX "Initialized the dma mmio va=%p\n", mmio_va_base);
	// Using this to check where the DMA lib is at for now.
	currentOwner = mmio_va_base == 0 ? MIC_DMA_CHAN_MIC_OWNED : MIC_DMA_CHAN_HOST_OWNED;

	// TODO: multi-card support
	md_mic_dma_init(&dma_ctx->dma_dev, mmio_va_base);

	for (i = 0 ; i < MAX_NUM_DMA_CHAN; i++) {
		ch = &dma_ctx->dma_channels[i];

		/* Initialize pointer to parent */
		ch->dma_ctx = dma_ctx;

		owner = i > __LAST_HOST_CHAN_NUM ? MIC_DMA_CHAN_MIC_OWNED
						: MIC_DMA_CHAN_HOST_OWNED;

		// This has to be done from card side
		ch->chan = md_mic_dma_request_chan(&dma_ctx->dma_dev, owner);
		KASSERT((ch->chan != NULL), "dummy\n");
		ch->ch_num = ch->chan->ch_num;

#ifdef _MIC_SCIF_
		/*
		 * Host driver would have executed by now and thus setup the
		 * desc. ring
		 */
		if (ch->chan->owner == MIC_DMA_CHAN_HOST_OWNED)
			md_mic_dma_enable_chan(&dma_ctx->dma_dev, i, true);
#endif

		atomic_set(&(ch->flags), CHAN_INUSE); // Mark as used by default
		if (currentOwner == owner) {
			alloc_dma_desc_ring_mem(ch, dma_ctx);

#ifdef _MIC_SCIF_ // DMA now shares the IRQ handler with other system interrupts
			ret_value = request_irq(i, dma_interrupt_handler, IRQF_DISABLED,
						"dma channel", ch);
			ret_value = ret_value;
			//pr_debug(PR_PREFIX "Interrupt handler ret value for  chan %d = %d\n", i, ret_value);
#endif
			md_mic_dma_chan_setup(dma_ctx, ch);

			mi_mic_dma_chan_setup(ch, dma_ctx);

			init_waitqueue_head(&ch->intr_wq);
			init_waitqueue_head(&ch->access_wq);
			// Only mark owned channel to be available
			atomic_set(&(ch->flags), CHAN_AVAILABLE);
			md_mic_dma_print_debug(&dma_ctx->dma_dev, ch->chan);
		} else {
			ch->desc_ring = NULL;
		}
	}

	/* Initialize last_allocated_dma_channel */
	dma_ctx->last_allocated_dma_channel_num = -1;
	//pr_debug(PR_PREFIX "Initialized the dma channels\n");
	mic_dma_proc_init(dma_ctx);
	return 0;
}

void
mic_dma_lib_uninit(struct mic_dma_ctx_t *dma_ctx)
{
	int i;
	struct dma_channel *ch;
#ifndef _MIC_SCIF_
	struct pci_dev *pdev;
#endif

	mic_dma_proc_uninit(dma_ctx);
	for (i = 0 ; i < MAX_NUM_DMA_CHAN; i++) {
		ch = &dma_ctx->dma_channels[i];
		if (!ch->desc_ring)
			continue;
		drain_dma_intr(ch);
		/* Request the channel but don't free it. Errors are okay */
		request_dma_channel(ch);
#ifdef _MIC_SCIF_ // DMA now shares the IRQ handler with other system interrupts
		free_irq(i, ch);
#endif
		mi_mic_dma_chan_destroy(ch, dma_ctx);
#ifndef _MIC_SCIF_
		micscif_pci_dev(dma_ctx->device_num, &pdev);
		mic_unmap_single(dma_ctx->device_num - 1, pdev, ch->desc_ring_phys,
			(DMA_DESC_RING_SIZE * sizeof(*ch->desc_ring)) + PAGE_SIZE);
#endif

        kfree(ch->desc_ring_bak);
		ch->desc_ring_bak = NULL; 
		ch->desc_ring = NULL;
		if (mic_hw_family(ch->dma_ctx->device_num) == FAMILY_KNC &&
			mic_hw_stepping(dma_ctx->device_num) >= KNC_B0_STEP) {
#ifndef _MIC_SCIF_
			mic_unmap_single(dma_ctx->device_num - 1, pdev, ch->chan->dstat_wb_phys,
				sizeof(uint32_t));
#endif
			kfree(ch->chan->dstat_wb_loc);
			ch->chan->dstat_wb_loc = NULL;
			ch->chan->dstat_wb_phys = 0;
		}
		md_mic_dma_free_chan(&dma_ctx->dma_dev, ch->chan);
	}
#ifndef MIC_IS_EMULATION
	/* Ensure that all waiters for DMA channels time out */
	msleep(DMA_TO/HZ * 1000);
#endif
	md_mic_dma_uninit(&dma_ctx->dma_dev);
	//pr_debug(PR_PREFIX "Uninitialized the dma channels\n");
}

/*
 * reserve_dma_channel - reserve a given dma channel for exclusive use
 *
 * @dma_handle - handle to DMA device returned by open_dma_device
 * @chan_num - Channel number to be reserved
 * @chan     - set to point to the dma channel reserved by the call
 *
 * Returns < 1 on error (errorno)
 * Returns 0 on success
 *
 * NOTES: Should this function sleep waiting for the lock?
 * TODO:
 * Maybe there should be a blocking and non-blocking versions of this function
 */
int
reserve_dma_channel(mic_dma_handle_t dma_handle, int chan_num, struct dma_channel **chan)
{
	struct mic_dma_ctx_t *dma_ctx = (struct mic_dma_ctx_t *) dma_handle;

	/*
	 * Do we need to do acquire the lock for statically allocated channels?
	 * I am assuming we dont have to lock
	 */
	if (CHAN_AVAILABLE == atomic_cmpxchg(&(dma_ctx->dma_channels[chan_num].flags), 
						   CHAN_AVAILABLE, CHAN_INUSE)) {
		*chan = &dma_ctx->dma_channels[chan_num];
		return 0;
	}
	return -1;
}
EXPORT_SYMBOL(reserve_dma_channel);

/*
 * allocate_dma_channel - dynamically allocate a dma channel (for a short while). Will
 *    search for, choose, and lock down one channel for use by the calling thread.
 *
 * @dma_handle - handle to DMA device returned by open_dma_device
 * @chan - Returns the dma_channel pointer that was allocated by the call
 *
 * Returns < 1 on error
 * Returns 0 on success
 *
 * NOTE:  This function grabs a lock before exiting -- the calling thread MUST NOT
 *  sleep, and must call free_dma_channel before returning to user-space or switching
 *  volantarily to another thread.  Similarly, this function cannot be called from
 *  an interrupt context at this time.
 *
 *  TODO: How do we pick a dma channel?
 *  For now I am doing it in round robin fashion.
 */
int
allocate_dma_channel(mic_dma_handle_t dma_handle, struct dma_channel **chan)
{
	struct mic_dma_ctx_t *dma_ctx = (struct mic_dma_ctx_t *) dma_handle;
	int i, j;

	if (!dma_ctx)
		return -ENODEV;

	j = dma_ctx->last_allocated_dma_channel_num + 1;

	for (i = 0; i < MAX_NUM_DMA_CHAN; i++, j++) {
		if (CHAN_AVAILABLE == atomic_cmpxchg(&(dma_ctx->dma_channels[j %
							     MAX_NUM_DMA_CHAN].flags), 
							   CHAN_AVAILABLE, CHAN_INUSE)) {
			*chan = &(dma_ctx->dma_channels[j % MAX_NUM_DMA_CHAN]);
			dma_ctx->last_allocated_dma_channel_num = j % MAX_NUM_DMA_CHAN;
			return 0;
		}
	}
	return -1;
}
EXPORT_SYMBOL(allocate_dma_channel);

/*
 * request_dma_channel - Request a specific DMA channel.
 *
 * @chan - Returns the dma_channel pointer that was requested
 *
 * Returns: 0 on success and -ERESTARTSYS if the wait was interrupted
 * or -EBUSY if the channel was not available.
 *
 * NOTE:  This function must call free_dma_channel before returning to
 * user-space.
 */
int request_dma_channel(struct dma_channel *chan)
{
	int ret;

	ret = wait_event_interruptible_timeout(chan->access_wq, 
		CHAN_AVAILABLE == atomic_cmpxchg(&chan->flags, 
		CHAN_AVAILABLE, CHAN_INUSE), DMA_TO);
	if (!ret) {
		printk(KERN_ERR "%s %d TO chan 0x%x\n", __func__, __LINE__, chan->ch_num);
		ret = -EBUSY;
	}
	if (ret > 0)
		ret = 0;
	return ret;
}
EXPORT_SYMBOL(request_dma_channel);

/*
 * free_dma_channel - after allocating a channel, used to
 *                 free the channel after DMAs are submitted
 *
 * @chan - pointer to the dma_channel struct that was allocated
 *
 * Returns 0 on success, < 1 on error (errorno)
 *
 * NOTE: This function must be called after all do_dma calls are finished,
 *  but can be called before the DMAs actually complete (as long as the comp_cb()
 *  handler in do_dma don't refer to the dma_channel struct).  If called with a
 *  dynamically allocated dma_chan, the caller must be the thread that called
 *  allocate_dma_chan.  When operating on a dynamic channel, free unlocks the
 *  mutex locked in allocate.  Statically allocated channels cannot be freed,
 *  and calling this function with that type of channel will return an error.
 */
int
free_dma_channel(struct dma_channel *chan)
{
	/*
	 * Why can't we use this function with channels that were statically allocated??
	 */
	BUG_ON(CHAN_INUSE !=
		atomic_cmpxchg(&chan->flags, CHAN_INUSE, CHAN_AVAILABLE));
	wake_up(&chan->access_wq);
	return 0;
}
EXPORT_SYMBOL(free_dma_channel);

static __always_inline uint32_t
get_dma_tail_pointer(struct dma_channel *chan)
{
	struct mic_dma_device *dma_dev;
	dma_dev = &chan->dma_ctx->dma_dev;
	return md_mic_dma_chan_read_tail(dma_dev, chan->chan);
}
/*
 * Return -1 in case of error
 */
static int
program_memcpy_descriptors(struct dma_channel *chan, uint64_t src, uint64_t dst, size_t len)
{
	size_t current_transfer_len;
	bool is_astep = false;
	unsigned long ts = jiffies;

	if (mic_hw_family(chan->dma_ctx->device_num) == FAMILY_KNC) {
		if (mic_hw_stepping(chan->dma_ctx->device_num) == KNC_A_STEP)
			is_astep = true;
	} else {
		is_astep = true;
	}
	do {
		current_transfer_len = (len > MAX_DMA_XFER_SIZE) ?
					MAX_DMA_XFER_SIZE : len;

		ts = jiffies;
		while (!md_avail_desc_ring_space(&chan->dma_ctx->dma_dev, is_astep, chan->chan,
						  (uint32_t)chan->next_write_index, 1)) {
				if (time_after(jiffies,ts + DMA_TO)) {
					printk(KERN_ERR "%s %d TO chan 0x%x\n", __func__, __LINE__, chan->ch_num);
					return -ENOMEM;
				}
		}

		//pr_debug("src_phys=0x%llx, dst_phys=0x%llx, size=0x%zx\n", src_phys_addr, dst_phys_addr, current_transfer_len);
		md_mic_dma_memcpy_desc(&chan->desc_ring[chan->next_write_index],
					    src, dst, current_transfer_len);
		chan->next_write_index = incr_rb_index((int)chan->next_write_index,
						       chan->chan->num_desc_in_ring);
		len -= current_transfer_len;
		dst = dst + current_transfer_len;
		src = src + current_transfer_len;
	} while(len > 0);

	return 0;
}

/*
 * do_dma - main dma function:  perform a dma memcpy, len bytes from src to dst
 *
 * @chan    - DMA channel to use for the transfer.  The channel can be allocated
 *            dynamically by calling allocate_dma_chan, or statically by
 *            reserve_dma_chan.  Using a channel not allocated in this way will
 *            result in undefined behavior.
 * @flags   - ATOMIC, called from an interrupt context (no blocking)
 * @src     - src physical address
 * @dst     - dst physical address
 * @len     - Length of the dma
 * @comp_cb - When the DMA is complete, the struct's function will be called.  NOTE!
 *            comp_cb(cb_cookie) is called from an interrupt context, so the
 *            function must not sleep or block.
 *
 * TODO: Figure out proper value instead of -2
 * Return < 0 on error
 * Return = -2 copy was done successfully, no need to wait
 * Return >= 0: DMA has been queued.  Return value can be polled on for completion
 *   if DO_DMA_POLLING was sent in flags
 *   (poll cookie).  An example (simplified w/ no error handling).
 *              int cookie = do_dma(...);
 *              while (poll_dma_completion(cookie) == 0);
 *              printf("DMA now complete\n");
 */
int
do_dma(struct dma_channel *chan, int flags, uint64_t src,
       uint64_t dst, size_t len, struct dma_completion_cb *comp_cb)
{
	/*
	 * TODO:
	 * Do we need to assert the ownership of channel??
	 */
	int poll_ring_index = -1;
	int intr_ring_index = -1;
	uint32_t num_status_desc = 0;
	bool is_astep = false;
	unsigned long ts = jiffies;

	might_sleep();
	if (flags & DO_DMA_INTR && !comp_cb)
		return -EINVAL;

	if (!verify_next_write_index(chan))
		return -ENODEV;

	//pr_debug(PR_PREFIX "Current transfer src = 0x%llx,dst = 0x%llx, len = 0x%zx\n", src, dst, len);
	if (flags & DO_DMA_INTR) {
		int err;
		err = wait_event_interruptible_timeout(chan->intr_wq, 
		(-1 != (intr_ring_index = allocate_buffer(&chan->intr_ring.ring))), 
		DMA_TO);
		if (!err) {
			printk(KERN_ERR "%s %d TO chan 0x%x\n", __func__, __LINE__, chan->ch_num);
			err = -ENOMEM;
		}
		if (err > 0)
			err = 0;
		if (!err) {
			chan->intr_ring.comp_cb_array[intr_ring_index] = comp_cb;
			num_status_desc++;
#ifdef CONFIG_MK1OM
			num_status_desc++;
#endif
		} else {
			return err;
		}
		//pr_debug(PR_PREFIX "INTR intr_ring_index=%d, chan_num=%lx\n", intr_ring_index, (chan - dma_channels));
	}

	if (flags & DO_DMA_POLLING) {
		poll_ring_index = allocate_buffer(&chan->poll_ring);
		if (-1 == poll_ring_index)
			return -ENOMEM;
		num_status_desc++;
		//pr_debug(PR_PREFIX "polling poll_ring_index=%d\n", poll_ring_index);
	}
	if (len && -ENOMEM == program_memcpy_descriptors(chan, src, dst, len)) {
		//pr_debug(PR_PREFIX "ERROR: do_dma: No available space from program_memcpy_descriptors\n");
		return -ENOMEM;
	}

	if (mic_hw_family(chan->dma_ctx->device_num) == FAMILY_KNC) {
		if (mic_hw_stepping(chan->dma_ctx->device_num) == KNC_A_STEP)
			is_astep = true;
	} else {
		is_astep = true;
	}

	ts = jiffies;

	while (num_status_desc && num_status_desc > md_avail_desc_ring_space(&chan->dma_ctx->dma_dev,
						is_astep, chan->chan, (uint32_t)chan->next_write_index, num_status_desc)) {
		if (time_after(jiffies,ts + DMA_TO)) {
			printk(KERN_ERR "%s %d TO chan 0x%x\n", __func__, __LINE__, chan->ch_num);
			return -ENOMEM;
		}
		//pr_debug(PR_PREFIX "ERROR: do_dma: No available space from md_avail_desc_ring_space\n");
	}

	if (flags & DO_DMA_POLLING) {
		incr_head(&chan->poll_ring);
		md_mic_dma_prep_status_desc(&chan->desc_ring[chan->next_write_index],
				poll_ring_index,
				chan->poll_ring.tail_phys,
				false);
		chan->next_write_index = incr_rb_index((int)chan->next_write_index,
						       chan->chan->num_desc_in_ring);
	}

	if (flags & DO_DMA_INTR) {
		incr_head(&chan->intr_ring.ring);
#ifdef CONFIG_MK1OM
		md_mic_dma_prep_status_desc(&chan->desc_ring[chan->next_write_index],
				intr_ring_index,
				chan->intr_ring.ring.tail_phys,
				false);
		chan->next_write_index = incr_rb_index((int)chan->next_write_index,
						       chan->chan->num_desc_in_ring);
#endif
		md_mic_dma_prep_status_desc(&chan->desc_ring[chan->next_write_index],
				intr_ring_index,
				chan->intr_ring.ring.tail_phys,
				true);
		chan->next_write_index = incr_rb_index((int)chan->next_write_index,
						       chan->chan->num_desc_in_ring);
	}

	/*
	 * TODO:
	 * Maybe it is better if we update the head pointer for every descriptor??
	 */
	md_mic_dma_chan_write_head(&chan->dma_ctx->dma_dev, chan->chan, (uint32_t)chan->next_write_index);
	//pr_debug(PR_PREFIX "in HW chan->next_write_index=%lld\n", chan->next_write_index);

	if (DO_DMA_POLLING & flags)
		return poll_ring_index;
	return 0;
}
EXPORT_SYMBOL(do_dma);

/*
 * poll_dma_completion - check if a DMA is complete
 *
 * @poll_cookie - value returned from do_dma
 *
 * Returns
 * 0 -> DMA pending
 * 1 -> DMA completed
 *
 * Note: This is mostly useful after calling do_dma with a NULL comp_cb parameter, as
 *  it will allow the caller to wait for DMA completion.
 */
int
poll_dma_completion(int poll_cookie, struct dma_channel *chan)
{
	if (!chan)
		return -EINVAL;
	/*
	 * In case of interrupts the ISR runs and reads the value
	 * of the tail location. If we are polling then we need
	 * to read the value of the tail location before checking
	 * if the entry is processed.
	 */
	chan->poll_ring.tail = read_tail(&chan->poll_ring);
	return is_entry_processed(&chan->poll_ring, poll_cookie);
}
EXPORT_SYMBOL(poll_dma_completion);

/*
 * do_status_update: Update physical address location with the value provided.
 *		Ensures all previous DMA descriptors submitted on this DMA
 *		channel are executed.
 * @chan    - DMA channel to use for the transfer. The channel can be allocated
 *            dynamically by calling allocate_dma_channel, or statically by
 *            reserve_dma_channel. Using a channel not allocated in this way will
 *            result in undefined behavior.
 * @phys    - physical address
 * @value   - Value to be programmed
 */
int do_status_update(struct dma_channel *chan, uint64_t phys, uint64_t value)
{
	unsigned long ts = jiffies;
	bool is_astep = false;

	if (!verify_next_write_index(chan))
		return -ENODEV;

	if (mic_hw_family(chan->dma_ctx->device_num) == FAMILY_KNC) {
		if (mic_hw_stepping(chan->dma_ctx->device_num) == KNC_A_STEP)
			is_astep = true;
	} else {
		is_astep = true;
	}
	/*
	 * TODO:
	 * Do we need to assert the ownership of channel??
	 */
	ts = jiffies;
	while (!md_avail_desc_ring_space(&chan->dma_ctx->dma_dev,
		is_astep, chan->chan, (uint32_t) chan->next_write_index, 1)) {
		cpu_relax();
		if (time_after(jiffies,ts + DMA_TO)) {
			printk(KERN_ERR "%s %d TO chan 0x%x\n", __func__, __LINE__, chan->ch_num);
			return -EBUSY;
		}
	}

	md_mic_dma_prep_status_desc(&chan->desc_ring[chan->next_write_index],
			value,
			phys,
			false);

	chan->next_write_index = incr_rb_index((int)chan->next_write_index,
			chan->chan->num_desc_in_ring);

	md_mic_dma_chan_write_head(&chan->dma_ctx->dma_dev,
			chan->chan, (uint32_t)chan->next_write_index);
	return 0;
}
EXPORT_SYMBOL(do_status_update);

/*
 * get_dma_mark: Obtain current value of DMA mark
 * @chan - DMA channel to use for the transfer. The channel can be allocated
 *            dynamically by calling allocate_dma_channel, or statically by
 *            reserve_dma_channel. Using a channel not allocated in this way will
 *            result in undefined behavior.
 */
int get_dma_mark(struct dma_channel *chan)
{
	if (chan)
		return chan->intr_ring.ring.head;
	else
		return -1;
}
EXPORT_SYMBOL(get_dma_mark);

/*
 * program_dma_mark: Increment the current value of the DMA mark for a DMA channel
 * and program an interrupt status update descriptor which ensures that all DMA
 * descriptors programmed uptil this point in time are completed.
 * @chan - DMA channel to use for the transfer. The channel can be allocated
 *            dynamically by calling allocate_dma_channel, or statically by
 *            reserve_dma_channel. Using a channel not allocated in this way will
 *            result in undefined behavior.
 */
int program_dma_mark(struct dma_channel *chan)
{
	/*
	 * TODO:
	 * Do we need to assert the ownership of channel??
	 */
	int intr_ring_index;
	int err;
	unsigned long ts = jiffies;
	uint32_t num_status_desc = 1;
	bool is_astep = false;

	if (!verify_next_write_index(chan))
		return -ENODEV;

	if (mic_hw_family(chan->dma_ctx->device_num) == FAMILY_KNC) {
		if (mic_hw_stepping(chan->dma_ctx->device_num) == KNC_A_STEP)
			is_astep = true;
	} else {
		is_astep = true;
	}
	might_sleep();
	err = wait_event_interruptible_timeout(chan->intr_wq, 
	(-1 != (intr_ring_index = allocate_buffer(&chan->intr_ring.ring))), 
	DMA_TO);
	if (!err)
		err = -EBUSY;
	if (err > 0)
		err = 0;
	if (err)
		return err;

#ifdef CONFIG_MK1OM
	num_status_desc++;
#endif
	ts = jiffies;
	while (num_status_desc > md_avail_desc_ring_space(&chan->dma_ctx->dma_dev,
			is_astep, chan->chan, (uint32_t)chan->next_write_index, num_status_desc)) {
		cpu_relax();
		if (time_after(jiffies,ts + DMA_TO)) {
			printk(KERN_ERR "%s %d TO chan 0x%x\n", __func__, __LINE__, chan->ch_num);
			return -EBUSY;
		}
	}

	chan->intr_ring.comp_cb_array[intr_ring_index] = NULL;

	incr_head(&chan->intr_ring.ring);
#ifdef CONFIG_MK1OM
	md_mic_dma_prep_status_desc(&chan->desc_ring[chan->next_write_index],
			intr_ring_index,
			chan->intr_ring.ring.tail_phys,
			false);
	chan->next_write_index = incr_rb_index((int)chan->next_write_index,
			chan->chan->num_desc_in_ring);
#endif
	md_mic_dma_prep_status_desc(&chan->desc_ring[chan->next_write_index],
			intr_ring_index,
			chan->intr_ring.ring.tail_phys,
			true);
	chan->next_write_index = incr_rb_index((int)chan->next_write_index,
			chan->chan->num_desc_in_ring);

	md_mic_dma_chan_write_head(&chan->dma_ctx->dma_dev, chan->chan, (uint32_t)chan->next_write_index);
	return intr_ring_index;
}
EXPORT_SYMBOL(program_dma_mark);

/*
 * is_current_dma_mark: Check if the dma mark provided is the current DMA mark.
 * @chan - DMA channel
 * @mark - DMA mark
 *
 * Return true on success and false on failure.
 */
bool is_current_dma_mark(struct dma_channel *chan, int mark)
{
	return (get_dma_mark(chan) == mark);
}
EXPORT_SYMBOL(is_current_dma_mark);

/*
 * is_dma_mark_processed: Check if the dma mark provided has been processed.
 * @chan - DMA channel
 * @mark - DMA mark
 *
 * Return true on success and false on failure.
 */
bool is_dma_mark_processed(struct dma_channel *chan, int mark)
{
	return is_entry_processed(&chan->intr_ring.ring, mark);
}
EXPORT_SYMBOL(is_dma_mark_processed);

/*
 * dma_mark_wait:
 * @chan - DMA channel
 * @mark - DMA mark
 * @is_interruptible - Use wait_event_interruptible() or not.
 *
 * Wait for the dma mark to complete.
 * Return 0 on success and appropriate error value on error.
 */
int dma_mark_wait(struct dma_channel *chan, int mark, bool is_interruptible)
{
	int err = 0;
	uint32_t prev_tail = 0, new_tail;
	uint32_t count = 0;

	if (chan) {
		might_sleep();
__retry:
		if (is_interruptible)
			err = wait_event_interruptible_timeout(
				chan->intr_wq, 
				is_dma_mark_processed(chan, mark), 
				DMA_TO);
		else
			err = wait_event_timeout(chan->intr_wq, 
				is_dma_mark_processed(chan, mark), DMA_TO);

		if (!err) { // 0 is timeout
			new_tail = get_dma_tail_pointer(chan);
			if ((count <= DMA_FENCE_TIMEOUT_CNT) &&
				(!count || new_tail != prev_tail)) {  // For performance, prev_tail is not read at the begining
					prev_tail = new_tail;
					count++;
					pr_debug("DMA fence wating is still ongoing, waiting for %d seconds\n", DMA_TO/HZ *count);
					goto __retry;
			} else {
				printk(KERN_ERR "%s %d TO chan 0x%x\n", __func__, __LINE__, chan->ch_num);
				err = -EBUSY;
			}
		}
		if (err > 0)
			err = 0;
	}
	return err;
}
EXPORT_SYMBOL(dma_mark_wait);

/*
 * drain_dma_poll - Drain all outstanding DMA operations for a particular
 * DMA channel via polling.
 * @chan - DMA channel
 * Return 0 on success and -errno on error.
 */
int drain_dma_poll(struct dma_channel *chan)
{
	int cookie, err;
	unsigned long ts;
	uint32_t prev_tail = 0, new_tail, count = 0;
	if (chan) {
		if ((err = request_dma_channel(chan)))
			goto error;
		if ((cookie = do_dma(chan,
			DO_DMA_POLLING, 0, 0, 0, NULL)) < 0) {
			err = cookie;
			free_dma_channel(chan);
			goto error;
		}
		free_dma_channel(chan);
		ts = jiffies;
		while (1 != poll_dma_completion(cookie, chan)) {
			cpu_relax();
			if (time_after(jiffies,ts + DMA_TO)) {
				new_tail = get_dma_tail_pointer(chan);
				if ((!count || new_tail != prev_tail) && (count <= DMA_FENCE_TIMEOUT_CNT)) {
					prev_tail = new_tail;
					ts = jiffies;
					count++;
					pr_debug("polling DMA is still ongoing,  wating for %d seconds\n", DMA_TO/HZ * count);
				} else {
					err = -EBUSY;
					break;
				}
			}
		}
error:
		if (err)
			printk(KERN_ERR "%s %d err %d\n", __func__, __LINE__, err);
	} else {
		err = -EINVAL;
	}
	return err;
}
EXPORT_SYMBOL(drain_dma_poll);

/*
 * drain_dma_intr - Drain all outstanding DMA operations for a particular
 * DMA channel via interrupt based blocking wait.
 * @chan - DMA channel
 * Return 0 on success and -errno on error.
 */
int drain_dma_intr(struct dma_channel *chan)
{
	int cookie, err;

	if (chan) {
		if ((err = request_dma_channel(chan)))
			goto error;
		if ((cookie = program_dma_mark(chan)) < 0) {
			err = cookie;
			free_dma_channel(chan);
			goto error;
		}
		free_dma_channel(chan);
		err = dma_mark_wait(chan, cookie, false);
error:
		if (err)
			printk(KERN_ERR "%s %d err %d\n", __func__, __LINE__, err);
	} else {
		err = -EINVAL;
	}
	return err;
}
EXPORT_SYMBOL(drain_dma_intr);

/*
 * drain_dma_global - Drain all outstanding DMA operations for
 * all online DMA channel.
 * Return none
 */
int drain_dma_global(mic_dma_handle_t dma_handle)
{
	int i, err = -EINVAL;
	struct dma_channel *chan;
	struct mic_dma_ctx_t *dma_ctx = (struct mic_dma_ctx_t *)dma_handle;

	if (!dma_ctx)
		return err;

	might_sleep();
	for (i = 0 ; i < MAX_NUM_DMA_CHAN; i++) {
		chan = &dma_ctx->dma_channels[i];
		if (chan->desc_ring == NULL)
			continue;
		if ((err = drain_dma_intr(chan)))
			break;
	}
	return err;
}
EXPORT_SYMBOL(drain_dma_global);

#ifdef _MIC_SCIF_
/*
 * dma_suspend: DMA tasks before transition to low power state.
 * @dma_handle: Handle for a DMA driver context.
 *
 * Perform the following tasks before the device transitions
 * to a low power state:
 * 1) Store away the DMA descriptor ring physical address base for
 * all DMA channels (both host/uOS owned) since the value would be
 * required to reinitialize the DMA channels upon transition from
 * low power to active state.
 *
 * Return: none
 * Notes: Invoked only on MIC.
 */
void dma_suspend(mic_dma_handle_t dma_handle)
{
	int i;
	struct dma_channel *ch;
	struct mic_dma_ctx_t *dma_ctx = (struct mic_dma_ctx_t *)dma_handle;
	struct mic_dma_device *dma_dev = &dma_ctx->dma_dev;

	for (i = 0; i < MAX_NUM_DMA_CHAN; i++) {
		ch = &dma_ctx->dma_channels[i];
		ch->desc_ring_phys =
			md_mic_dma_chan_get_desc_ring_phys(dma_dev, ch->chan);
		ch->chan->dstat_wb_phys =
			md_mic_dma_chan_get_dstatwb_phys(dma_dev, ch->chan);
	}
}
EXPORT_SYMBOL(dma_suspend);

/*
 * dma_resume: DMA tasks after wake up from low power state.
 * @dma_handle: Handle for a DMA driver context.
 *
 * Performs the following tasks before the device transitions
 * from a low power state to active state:
 * 1) As a test, reset the value in DMA configuration register.
 * 2) Reset the next_write_index for the DMA descriptor ring to 0
 * since the DMA channel will be reset shortly.
 * 3) Reinitialize the DMA MD layer for the channel.
 *
 * Return: none
 * Notes:
 * Notes: Invoked only on MIC.
 */
void dma_resume(mic_dma_handle_t dma_handle)
{
	int i;
	struct dma_channel *ch;
	struct mic_dma_ctx_t *dma_ctx = (struct mic_dma_ctx_t *)dma_handle;
	struct mic_dma_device *dma_dev = &dma_ctx->dma_dev;

	/* TODO: Remove test write to SBOX_DCR */
	mic_sbox_write_mmio(dma_dev->mm_sbox, SBOX_DCR, 0);
	for (i = 0; i < MAX_NUM_DMA_CHAN; i++) {
		ch = &dma_ctx->dma_channels[i];
		ch->next_write_index = 0;
		md_mic_dma_chan_init_attr(dma_dev, ch->chan);
		md_mic_dma_chan_setup(dma_ctx, ch);
	}
}
EXPORT_SYMBOL(dma_resume);

#else

/*
 * dma_prep_suspend: DMA tasks required on host before a device can transition
 * to a low power state.
 * @dma_handle: Handle for a DMA driver context.
 *
 * Performs the following tasks on the host before the device can be allowed
 * to transiti to a low power state.
 * 1) Reset the next_Write_index for the DMA descriptor ring to 0
 * since the DMA channel will be reset shortly. This is required primarily
 * for Host owned DMA channels since MIC does not have access to this
 * information.
 * Return: none
 * Invoked only on Host.
 */
void dma_prep_suspend(mic_dma_handle_t dma_handle)
{
	int i;
	struct dma_channel *ch;
	struct mic_dma_ctx_t *dma_ctx = (struct mic_dma_ctx_t *)dma_handle;

	for (i = 0; i < MAX_NUM_DMA_CHAN; i++) {
		ch = &dma_ctx->dma_channels[i];
		ch->next_write_index = 0;
	}
}
EXPORT_SYMBOL(dma_prep_suspend);
#endif

#ifdef CONFIG_PAGE_CACHE_DMA
#ifdef _MIC_SCIF_
static const struct dma_operations dma_operations_fast_copy = {
	.do_dma               = do_dma,
	.poll_dma_completion  = poll_dma_completion,
	.free_dma_channel     = free_dma_channel,
	.open_dma_device      = open_dma_device,
	.close_dma_device     = close_dma_device,
	.allocate_dma_channel = allocate_dma_channel,
	.program_descriptors  = program_memcpy_descriptors,
	.do_dma_polling				= DO_DMA_POLLING,
};

static const struct file_dma fdma_callback = {
	.dmaops = &dma_operations_fast_copy,
};
#endif
#endif

#ifdef _MIC_SCIF_
static int
#else
int
#endif
mic_dma_init(void)
{
	int i;

	for (i = 0; i < MAX_BOARD_SUPPORTED; i++)
		mutex_init (&lock_dma_dev_init[i]);
#ifdef CONFIG_PAGE_CACHE_DMA
#ifdef _MIC_SCIF_
	register_dma_for_fast_copy(&fdma_callback);
#endif
#endif
	return 0;
}

#ifdef _MIC_SCIF_
static void mic_dma_uninit(void)
{
#ifdef CONFIG_PAGE_CACHE_DMA
	unregister_dma_for_fast_copy();
#endif
}

module_init(mic_dma_init);
module_exit(mic_dma_uninit);
#endif

#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0))
static int
mic_dma_proc_ring_show(struct seq_file *m, void *data)
{
	struct mic_dma_ctx_t *dma_ctx = m->private;
	mic_ctx_t *mic_ctx = get_per_dev_ctx(dma_ctx->device_num - 1);
	int i, err;
	struct compl_buf_ring *ring;

	if ((err = micpm_get_reference(mic_ctx, true))) {
		printk(KERN_ERR "%s %d: unable to get micpm reference: %d\n", 
				 __func__, __LINE__, err);
		return err;
	}

	seq_printf(m, "Intr rings\n");
	seq_printf(m, "%-10s%-12s%-12s%-12s%-25s%-18s%-25s\n",
		       "Chan", "Head", "Tail", "Size", "Tail loc", "Actual tail", "In Use");
	for (i = first_dma_chan(); i <= last_dma_chan(); i++) {
		ring = &dma_ctx->dma_channels[i].intr_ring.ring;
		seq_printf(m, "%-#10x%-#12x%-#12x%-#12x%-#25llx%-#18x%-#18x\n",
			i, ring->head, ring->tail, ring->size,
			ring->tail_location, *(int*)ring->tail_location,
			atomic_read(&dma_ctx->dma_channels[i].flags));
	}
	seq_printf(m, "Poll rings\n");
	seq_printf(m, "%-10s%-12s%-12s%-12s%-25s%-18s\n",
		       "Chan", "Head", "Tail", "Size", "Tail loc", "Actual tail");
	for (i = first_dma_chan(); i <= last_dma_chan(); i++) {
		ring = &dma_ctx->dma_channels[i].poll_ring;
		seq_printf(m, "%-#10x%-#12x%-#12x%-#12x%-#25llx%-#18x\n",
			       i, ring->head, ring->tail, ring->size,
			       ring->tail_location, *(int*)ring->tail_location);
	}
	seq_printf(m, "Next_Write_Index\n");
	seq_printf(m, "%-10s%-12s\n", "Chan", "Next_Write_Index");
	for (i = 0; i < MAX_NUM_DMA_CHAN; i++) {
		seq_printf(m, "%-#10x%-#12llx\n",
			       i, dma_ctx->dma_channels[i].next_write_index);
	}
	micpm_put_reference(mic_ctx);
	return 0;
}

static int
mic_dma_proc_ring_open(struct inode *inode, struct file *file)
{
	return single_open(file, mic_dma_proc_ring_show, PDE_DATA(inode));
}

static int
mic_dma_proc_reg_show(struct seq_file *m, void *data)
{
	int i, j, chan_num, size, dtpr, err;
	struct mic_dma_ctx_t *dma_ctx = m->private;
	mic_ctx_t *mic_ctx = get_per_dev_ctx(dma_ctx->device_num - 1);
	struct mic_dma_device *dma_dev = &dma_ctx->dma_dev;
	struct dma_channel *curr_chan;
	union md_mic_dma_desc desc;

	if ((err = micpm_get_reference(mic_ctx, true))) {
		printk(KERN_ERR "%s %d: unable to get micpm reference: %d\n", 
				 __func__, __LINE__, err);
		return err;
	}

	seq_printf(m, "========================================"
				"=======================================\n");
	seq_printf(m, "SBOX_DCR: %#x\n",
				mic_sbox_read_mmio(dma_dev->mm_sbox, SBOX_DCR));
	seq_printf(m, "DMA Channel Registers\n");
	seq_printf(m, "========================================"
				"=======================================\n");
	seq_printf(m, "%-10s| %-10s %-10s %-10s %-10s %-10s %-10s"
#ifdef CONFIG_MK1OM
				  " %-10s %-11s %-14s %-10s"
#endif
				"\n", "Channel", "DCAR", "DTPR", "DHPR",
					"DRAR_HI", "DRAR_LO",
#ifdef CONFIG_MK1OM
					"DSTATWB_LO", "DSTATWB_HI", "DSTAT_CHERR", "DSTAT_CHERRMSK",
#endif
					"DSTAT");
	seq_printf(m, "========================================"
				"=======================================\n");

#ifdef _MIC_SCIF_
	for (i = 0; i < MAX_NUM_DMA_CHAN; i++) {
#else
	for (i = first_dma_chan(); i <= last_dma_chan(); i++) {
#endif
		curr_chan = &dma_ctx->dma_channels[i];
		chan_num = curr_chan->ch_num;
		seq_printf(m, "%-10i| %-#10x %-#10x %-#10x %-#10x"
			" %-#10x"
#ifdef CONFIG_MK1OM
			" %-#10x %-#11x %-#10x %-#14x"
#endif
			" %-#10x\n", chan_num,
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCAR),
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DTPR),
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DHPR),
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DRAR_HI),
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DRAR_LO),
#ifdef CONFIG_MK1OM
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTATWB_LO),
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTATWB_HI),
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCHERR),
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCHERRMSK),
#endif
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTAT));
	}

	seq_printf(m, "\nDMA Channel Descriptor Rings\n");
	seq_printf(m, "========================================"
				"=======================================\n");

	for (i = first_dma_chan(); i <= last_dma_chan(); i++) {
		curr_chan = &dma_ctx->dma_channels[i];
		chan_num = curr_chan->ch_num;
		dtpr = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DTPR);
		seq_printf(m,  "Channel %i: [", chan_num);
		size = ((int) md_mic_dma_read_mmio(dma_dev, chan_num, REG_DHPR)
			- dtpr) % curr_chan->chan->num_desc_in_ring;
		/*
		 * In KNC B0, empty condition is tail = head -1
		 */
		if (mic_hw_family(dma_ctx->device_num) == FAMILY_KNC &&
			mic_hw_stepping(dma_ctx->device_num) >= KNC_B0_STEP)
			size -= 1;

		for (j = 0; j < size; j++) {
			desc = curr_chan->desc_ring[(j+dtpr) % 
				curr_chan->chan->num_desc_in_ring];	

			switch (desc.desc.nop.type){
			case NOP:
				seq_printf(m," {Type: NOP, 0x%#llx"
					" %#llx} ",  desc.qwords.qw0,
						   desc.qwords.qw1);
			case MEMCOPY:
				seq_printf(m," {Type: MEMCOPY, SAP:"
					" 0x%#llx, DAP: %#llx, length: %#llx} ",
					  (uint64_t) desc.desc.memcopy.sap,
					  (uint64_t) desc.desc.memcopy.dap,
					  (uint64_t) desc.desc.memcopy.length);
				break;
			case STATUS:
				seq_printf(m," {Type: STATUS, data:"
					" 0x%#llx, DAP: %#llx, intr: %lli} ",
					(uint64_t) desc.desc.status.data,
					(uint64_t) desc.desc.status.dap,
					(uint64_t) desc.desc.status.intr);
				break;
			case GENERAL:
				seq_printf(m," {Type: GENERAL, "
					"DAP: %#llx, dword: %#llx} ",
					(uint64_t) desc.desc.general.dap,
					(uint64_t) desc.desc.general.data);
				break;
			case KEYNONCECNT:
				seq_printf(m," {Type: KEYNONCECNT, sel: "
					"%lli, h: %lli, index: %lli, cs: %lli,"
					" value: %#llx} ",
						(uint64_t) desc.desc.keynoncecnt.sel,
						(uint64_t) desc.desc.keynoncecnt.h,
						(uint64_t) desc.desc.keynoncecnt.index,
						(uint64_t) desc.desc.keynoncecnt.cs,
						(uint64_t) desc.desc.keynoncecnt.data);
				break;
			case KEY:
				seq_printf(m," {Type: KEY, dest_ind"
					   "ex: %lli, ski: %lli, skap: %#llx ",
						(uint64_t) desc.desc.key.di,
						(uint64_t) desc.desc.key.ski,
						(uint64_t) desc.desc.key.skap);
				break;
			default:
				seq_printf(m," {Uknown Type=%lli ,"
				 "%#llx %#llx} ",(uint64_t)  desc.desc.nop.type,
						(uint64_t) desc.qwords.qw0,
						(uint64_t) desc.qwords.qw1);
			}
		}
		seq_printf(m,  "]\n");
		if (mic_hw_family(dma_ctx->device_num) == FAMILY_KNC &&
		    mic_hw_stepping(dma_ctx->device_num) >= KNC_B0_STEP &&
		    curr_chan->chan->dstat_wb_loc)
			seq_printf(m, "DSTAT_WB = 0x%x\n",
				*((uint32_t*)curr_chan->chan->dstat_wb_loc));
	}
	micpm_put_reference(mic_ctx);

	return 0;
}

static int
mic_dma_proc_reg_open(struct inode *inode, struct file *file)
{
	return single_open(file, mic_dma_proc_reg_show, PDE_DATA(inode));
}

struct file_operations micdma_ring_fops = {
	.open		= mic_dma_proc_ring_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
        .release 	= single_release,
};

struct file_operations micdma_reg_fops = {
	.open		= mic_dma_proc_reg_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
        .release 	= single_release,
};

static void
mic_dma_proc_init(struct mic_dma_ctx_t *dma_ctx)
{
	char name[64];

	snprintf(name, 63, "%s%d", proc_dma_ring, dma_ctx->device_num);
	if (!proc_create_data(name,  S_IFREG | S_IRUGO, NULL, &micdma_ring_fops, dma_ctx))
		printk("micdma: unable to register /proc/%s\n", name);

	snprintf(name, 63, "%s%d", proc_dma_reg, dma_ctx->device_num);
	if (!proc_create_data(name, S_IFREG | S_IRUGO, NULL, &micdma_reg_fops, dma_ctx))
		printk("micdma: unable to register /proc/%s\n", name);

}
#else // LINUX VERSION
static int
mic_dma_proc_read_fn(char *buf, char **start, off_t offset, int count, int *eof, void *data)
{
	struct mic_dma_ctx_t *dma_ctx = data;
	int i, len = 0;
	struct compl_buf_ring *ring;

	len += sprintf(buf + len, "Intr rings\n");
	len += sprintf(buf + len, "%-10s%-12s%-12s%-12s%-25s%-18s%-25s\n",
		       "Chan", "Head", "Tail", "Size", "Tail loc", "Actual tail", "In Use");
	for (i = first_dma_chan(); i <= last_dma_chan(); i++) {
		ring = &dma_ctx->dma_channels[i].intr_ring.ring;
		len += sprintf(buf + len, "%-#10x%-#12x%-#12x%-#12x%-#25llx%-#18x%-#18x\n",
			i, ring->head, ring->tail, ring->size,
			ring->tail_location, *(int*)ring->tail_location,
			atomic_read(&dma_ctx->dma_channels[i].flags));
	}
	len += sprintf(buf + len, "Poll rings\n");
	len += sprintf(buf + len, "%-10s%-12s%-12s%-12s%-25s%-18s\n",
		       "Chan", "Head", "Tail", "Size", "Tail loc", "Actual tail");
	for (i = first_dma_chan(); i <= last_dma_chan(); i++) {
		ring = &dma_ctx->dma_channels[i].poll_ring;
		len += sprintf(buf + len, "%-#10x%-#12x%-#12x%-#12x%-#25llx%-#18x\n",
			       i, ring->head, ring->tail, ring->size,
			       ring->tail_location, *(int*)ring->tail_location);
	}
	len += sprintf(buf + len, "Next_Write_Index\n");
	len += sprintf(buf + len, "%-10s%-12s\n", "Chan", "Next_Write_Index");
	for (i = 0; i < MAX_NUM_DMA_CHAN; i++) {
		len += sprintf(buf + len, "%-#10x%-#12llx\n",
			       i, dma_ctx->dma_channels[i].next_write_index);
	}
	return len;
}

static int
mic_dma_proc_read_registers_fn(char *buf, char **start, off_t offset, int count,
							   int *eof, void *data)
{
	int i, j, chan_num, size, dtpr, len = 0;
	struct mic_dma_ctx_t *dma_ctx = data;
	struct mic_dma_device *dma_dev = &dma_ctx->dma_dev;
	struct dma_channel *curr_chan;
	union md_mic_dma_desc desc;

	len += sprintf(buf + len, "========================================"
				"=======================================\n");
	len += sprintf(buf + len, "SBOX_DCR: %#x\n",
				mic_sbox_read_mmio(dma_dev->mm_sbox, SBOX_DCR));
	len += sprintf(buf + len, "DMA Channel Registers\n");
	len += sprintf(buf + len, "========================================"
				"=======================================\n");
	len += sprintf(buf + len, "%-10s| %-10s %-10s %-10s %-10s %-10s %-10s"
#ifdef CONFIG_MK1OM
				  " %-10s %-11s %-14s %-10s"
#endif
				"\n", "Channel", "DCAR", "DTPR", "DHPR",
					"DRAR_HI", "DRAR_LO",
#ifdef CONFIG_MK1OM
					"DSTATWB_LO", "DSTATWB_HI", "DSTAT_CHERR", "DSTAT_CHERRMSK",
#endif
					"DSTAT");
	len += sprintf(buf + len, "========================================"
				"=======================================\n");

#ifdef _MIC_SCIF_
	for (i = 0; i < MAX_NUM_DMA_CHAN; i++) {
#else
	for (i = first_dma_chan(); i <= last_dma_chan(); i++) {
#endif
		curr_chan = &dma_ctx->dma_channels[i];
		chan_num = curr_chan->ch_num;
		len += sprintf(buf + len, "%-10i| %-#10x %-#10x %-#10x %-#10x"
			" %-#10x"
#ifdef CONFIG_MK1OM
			" %-#10x %-#11x %-#10x %-#14x"
#endif
			" %-#10x\n", chan_num,
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCAR),
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DTPR),
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DHPR),
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DRAR_HI),
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DRAR_LO),
#ifdef CONFIG_MK1OM
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTATWB_LO),
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTATWB_HI),
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCHERR),
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCHERRMSK),
#endif
			md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTAT));
	}

	len += sprintf(buf + len, "\nDMA Channel Descriptor Rings\n");
	len += sprintf(buf + len, "========================================"
				"=======================================\n");

	for (i = first_dma_chan(); i <= last_dma_chan(); i++) {
		curr_chan = &dma_ctx->dma_channels[i];
		chan_num = curr_chan->ch_num;
		dtpr = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DTPR);
		len += sprintf(buf + len,  "Channel %i: [", chan_num);
		size = ((int) md_mic_dma_read_mmio(dma_dev, chan_num, REG_DHPR)
			- dtpr) % curr_chan->chan->num_desc_in_ring;
		/*
		 * In KNC B0, empty condition is tail = head -1
		 */
		if (mic_hw_family(dma_ctx->device_num) == FAMILY_KNC &&
			mic_hw_stepping(dma_ctx->device_num) >= KNC_B0_STEP)
			size -= 1;

		for (j = 0; j < size; j++) {
			desc = curr_chan->desc_ring[(j+dtpr) % 
				curr_chan->chan->num_desc_in_ring];	

			switch (desc.desc.nop.type){
			case NOP:
				len += sprintf(buf + len," {Type: NOP, 0x%#llx"
					" %#llx} ",  desc.qwords.qw0,
						   desc.qwords.qw1);
			case MEMCOPY:
				len += sprintf(buf + len," {Type: MEMCOPY, SAP:"
					" 0x%#llx, DAP: %#llx, length: %#llx} ",
					  (uint64_t) desc.desc.memcopy.sap,
					  (uint64_t) desc.desc.memcopy.dap,
					  (uint64_t) desc.desc.memcopy.length);
				break;
			case STATUS:
				len += sprintf(buf + len," {Type: STATUS, data:"
					" 0x%#llx, DAP: %#llx, intr: %lli} ",
					(uint64_t) desc.desc.status.data,
					(uint64_t) desc.desc.status.dap,
					(uint64_t) desc.desc.status.intr);
				break;
			case GENERAL:
				len += sprintf(buf + len," {Type: GENERAL, "
					"DAP: %#llx, dword: %#llx} ",
					(uint64_t) desc.desc.general.dap,
					(uint64_t) desc.desc.general.data);
				break;
			case KEYNONCECNT:
				len += sprintf(buf + len," {Type: KEYNONCECNT, sel: "
					"%lli, h: %lli, index: %lli, cs: %lli,"
					" value: %#llx} ",
						(uint64_t) desc.desc.keynoncecnt.sel,
						(uint64_t) desc.desc.keynoncecnt.h,
						(uint64_t) desc.desc.keynoncecnt.index,
						(uint64_t) desc.desc.keynoncecnt.cs,
						(uint64_t) desc.desc.keynoncecnt.data);
				break;
			case KEY:
				len += sprintf(buf + len," {Type: KEY, dest_ind"
					   "ex: %lli, ski: %lli, skap: %#llx ",
						(uint64_t) desc.desc.key.di,
						(uint64_t) desc.desc.key.ski,
						(uint64_t) desc.desc.key.skap);
				break;
			default:
				len += sprintf(buf + len," {Uknown Type=%lli ,"
				 "%#llx %#llx} ",(uint64_t)  desc.desc.nop.type,
						(uint64_t) desc.qwords.qw0,
						(uint64_t) desc.qwords.qw1);
			}
		}
		len += sprintf(buf + len,  "]\n");
		if (mic_hw_family(dma_ctx->device_num) == FAMILY_KNC &&
		    mic_hw_stepping(dma_ctx->device_num) >= KNC_B0_STEP &&
		    curr_chan->chan->dstat_wb_loc)
			len += sprintf(buf + len, "DSTAT_WB = 0x%x\n",
				*((uint32_t*)curr_chan->chan->dstat_wb_loc));
	}
	return len;
}

static void
mic_dma_proc_init(struct mic_dma_ctx_t *dma_ctx)
{
	struct proc_dir_entry *dma_proc;
	char name[64];

	snprintf(name, 63, "%s%d", proc_dma_ring, dma_ctx->device_num);
	if ((dma_proc = create_proc_entry(name,  S_IFREG | S_IRUGO, NULL)) != NULL) {
		dma_proc->read_proc = mic_dma_proc_read_fn;
		dma_proc->data      = dma_ctx;
	}
	snprintf(name, 63, "%s%d", proc_dma_reg, dma_ctx->device_num);
	if ((dma_proc = create_proc_entry(name, S_IFREG | S_IRUGO, NULL)) != NULL) {
		dma_proc->read_proc = mic_dma_proc_read_registers_fn;
		dma_proc->data      = dma_ctx;
	}

}
#endif // LINUX VERSION

static void
mic_dma_proc_uninit(struct mic_dma_ctx_t *dma_ctx)
{
	char name[64];

	snprintf(name, 63, "%s%d", proc_dma_reg, dma_ctx->device_num);
	remove_proc_entry(name, NULL);
	snprintf(name, 63, "%s%d", proc_dma_ring, dma_ctx->device_num);
	remove_proc_entry(name, NULL);
}