* Copyright 2010-2017 Intel Corporation.
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License, version 2,
* as published by the Free Software Foundation.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
* Disclaimer: The codes contained in these modules may be specific to
* the Intel Software Development Platform codenamed Knights Ferry,
* and the Intel product codenamed Knights Corner, and are not backward
* compatible with other Intel products. Additionally, Intel will NOT
* support the codes or instruction set in future products.
* Intel offers no warranty of any kind regarding the code. This code is
* licensed on an "AS IS" basis and Intel is not obligated to provide
* any support, assistance, installation, training, or other services
* of any kind. Intel is also not obligated to provide any updates,
* enhancements or extensions. Intel specifically disclaims any warranty
* of merchantability, non-infringement, fitness for any particular
* purpose, and any other warranty.
* Further, Intel disclaims all liability of any kind, including but
* not limited to liability for infringement of any proprietary rights,
* relating to the use of the code, even if Intel is notified of the
* possibility of such liability. Except as expressly stated in an Intel
* license agreement provided with this code and agreed upon with Intel,
* no license, express or implied, by estoppel or otherwise, to any
* intellectual property rights is granted herein.
#include<linux/interrupt.h>
#include<linux/proc_fs.h>
#include<linux/version.h>
#include <asm/mic/mic_common.h>
#ifdef CONFIG_PAGE_CACHE_DMA
#include <linux/mic_dma/mic_dma_callback.h>
#include <mic/mic_dma_lib.h>
#include <mic/micscif_smpt.h>
#include <mic/mic_dma_md.h>
#include <mic/mic_dma_api.h>
#include <mic/compl_buf_ring.h>
#include <mic/micscif_smpt.h>
#include <mic/micsboxdefine.h>
#define DMA_FENCE_TIMEOUT_CNT (INT_MAX)
#define DMA_SLOWEST_BW (300) // 300Mbps
// the maximum size for each decriptor entry is 2M
#define DMA_FENCE_TIMEOUT_CNT (2 * MIC_MAX_NUM_DESC_PER_RING /DMA_SLOWEST_BW/ (DMA_TO/HZ))
#define MAX_DMA_XFER_SIZE MIC_MAX_DMA_XFER_SIZE
/* Use 512K as the maximum descriptor transfer size for Host */
#define MAX_DMA_XFER_SIZE (((1U) * 1024 * 1024) >> 1)
#define KASSERT(x, y, ...) \
printk(y, ##__VA_ARGS__);\
* Arrary of per device DMA contexts. The card only uses index 0. The host uses one
* context per card starting from 0.
static struct mic_dma_ctx_t
*mic_dma_context
[MAX_BOARD_SUPPORTED
+ 1];
static struct mutex lock_dma_dev_init
[MAX_BOARD_SUPPORTED
+ 1];
enum mic_desc_format_type
{
char proc_dma_reg
[]="mic_dma_registers_";
char proc_dma_ring
[]="mic_dma_ring_";
#define PR_PREFIX "DMA_LIB_MI:"
#define DMA_DESC_RING_SIZE MIC_MAX_NUM_DESC_PER_RING
#define MAX_POLLING_BUFFERS DMA_DESC_RING_SIZE
static void mic_dma_proc_init(struct mic_dma_ctx_t
*dma_ctx
);
static void mic_dma_proc_uninit(struct mic_dma_ctx_t
*dma_ctx
);
* TODO: This is size of s/w interrupt ring.
* We need to figure out a value so that we don't run out of memory in
* interrupt ring and at the same time don't waste memory
#define NUM_COMP_BUFS (((PAGE_SIZE/sizeof(struct dma_completion_cb*)) - 10) * 10)
struct intr_compl_buf_ring
{
struct dma_completion_cb
**comp_cb_array
;
struct compl_buf_ring ring
;
struct mic_dma_ctx_t
; /* Forward Declaration */
int ch_num
;/*Duplicated in md_mic_dma_chan struct too*/
struct md_mic_dma_chan
*chan
;
wait_queue_head_t intr_wq
;
wait_queue_head_t access_wq
;
union md_mic_dma_desc
*desc_ring_bak
;
union md_mic_dma_desc
*desc_ring
;
phys_addr_t desc_ring_phys
;
uint64_t next_write_index
; /* next write index into desc ring */
struct intr_compl_buf_ring intr_ring
;
struct compl_buf_ring poll_ring
;
struct mic_dma_ctx_t
*dma_ctx
; /* Pointer to parent DMA context */
/* Per MIC device (per MIC board) DMA context */
struct dma_channel dma_channels
[MAX_NUM_DMA_CHAN
];
int last_allocated_dma_channel_num
;
struct mic_dma_device dma_dev
;
atomic_t ref_count
; /* Reference count */
/* DMA Library Init/Uninit Routines */
static int mic_dma_lib_init(uint8_t *mmio_va_base
, struct mic_dma_ctx_t
*dma_ctx
);
static void mic_dma_lib_uninit(struct mic_dma_ctx_t
*dma_ctx
);
int get_chan_num(struct dma_channel
*chan
)
EXPORT_SYMBOL(get_chan_num
);
void initdmaglobalvar(void)
memset(mic_dma_context
, 0, sizeof(struct mic_dma_ctx_t
*) * (MAX_BOARD_SUPPORTED
+ 1));
ack_dma_interrupt(struct dma_channel
*ch
)
md_mic_dma_chan_mask_intr(&ch
->dma_ctx
->dma_dev
, ch
->chan
);
md_mic_dma_chan_unmask_intr(&ch
->dma_ctx
->dma_dev
, ch
->chan
);
/* Returns true if the next write index is "within" bounds */
static inline bool verify_next_write_index(struct dma_channel
*ch
)
if (ch
->next_write_index
< DMA_DESC_RING_SIZE
)
printk(KERN_ERR
"%s %d OOB ch_num 0x%x next_write_index 0x%llx\n",
ch
->ch_num
, ch
->next_write_index
);
* See if we can use __get_free_pages or something similar
* get_free_pages expects a power of 2 number of pages
alloc_dma_desc_ring_mem(struct dma_channel
*ch
, struct mic_dma_ctx_t
*dma_ctx
)
/* Is there any kernel allocator which provides the
* option to give the alignment??
(DMA_DESC_RING_SIZE
* sizeof(*ch
->desc_ring
)) + PAGE_SIZE
, GFP_KERNEL
);
ch
->desc_ring_bak
= ch
->desc_ring
;
ch
->desc_ring
= (union md_mic_dma_desc
*)ALIGN(
(uint64_t)ch
->desc_ring
, PAGE_SIZE
);
ch
->desc_ring_phys
= virt_to_phys(ch
->desc_ring
);
micscif_pci_dev(dma_ctx
->device_num
, &pdev
);
ch
->desc_ring_phys
= mic_map_single(dma_ctx
->device_num
- 1, pdev
, (void *)ch
->desc_ring
,
(DMA_DESC_RING_SIZE
* sizeof(*ch
->desc_ring
)) + PAGE_SIZE
);
BUG_ON(pci_dma_mapping_error(pdev
, ch
->desc_ring_phys
));
* Call completion cb functions:
* Take care of case where we allocated temp buf
mic_dma_lib_interrupt_handler(struct dma_channel
*chan
)
int ring_size
= chan
->intr_ring
.ring
.size
;
struct dma_completion_cb
**temp
= chan
->intr_ring
.comp_cb_array
;
struct dma_completion_cb
*cb
;
if (mic_hw_family(chan
->dma_ctx
->device_num
) == FAMILY_KNC
&&
mic_hw_stepping(chan
->dma_ctx
->device_num
) >= KNC_B0_STEP
) {
unsigned long error
= *((uint32_t*)chan
->chan
->dstat_wb_loc
);
if (unlikely(test_bit(31, &error
)))
printk(KERN_ERR
"DMA h/w error - %s %d, dstatwb=%lx\n",
__func__
, __LINE__
, error
);
new_tail
= read_tail(&chan
->intr_ring
.ring
);
old_tail
= chan
->intr_ring
.old_tail
;
for (; i
< ring_size
&& old_tail
!= new_tail
;
old_tail
= incr_rb_index(old_tail
, ring_size
), i
++) {
cb
= (struct dma_completion_cb
*)xchg(&temp
[old_tail
], NULL
);
cb
->dma_completion_func(cb
->cb_cookie
);
chan
->intr_ring
.old_tail
= new_tail
;
update_tail(&chan
->intr_ring
.ring
, new_tail
);
if (i
== ring_size
&& old_tail
!= new_tail
) {
printk(KERN_ERR PR_PREFIX
"Something went wrong, old tail = %d, new tail = %d\n",
* Maybe move the logic into slow interrupt handler
dma_interrupt_handler(int irq
, void *dev_id
)
struct dma_channel
*chan
= ((struct dma_channel
*)dev_id
);
mic_dma_lib_interrupt_handler(chan
);
#define SBOX_SICR0_DMA(x) (((x) >> 8) & 0xff)
* Maybe move the logic into slow interrupt handler
host_dma_interrupt_handler(mic_dma_handle_t dma_handle
, uint32_t sboxSicr0reg
)
struct mic_dma_ctx_t
*dma_ctx
= (struct mic_dma_ctx_t
*) dma_handle
;
for (dma_chan_id
= 0; dma_chan_id
< 8; dma_chan_id
++) {
if (SBOX_SICR0_DMA(sboxSicr0reg
) & (0x1 << dma_chan_id
)) {
ch
= &dma_ctx
->dma_channels
[dma_chan_id
];
host_dma_lib_interrupt_handler(ch
);
host_dma_lib_interrupt_handler(struct dma_channel
*chan
)
mic_dma_lib_interrupt_handler(chan
);
mi_mic_dma_chan_setup(struct dma_channel
*ch
, struct mic_dma_ctx_t
*dma_ctx
)
ch
->next_write_index
= ch
->chan
->cached_tail
;
init_ring(&ch
->poll_ring
, MAX_POLLING_BUFFERS
, dma_ctx
->device_num
);
ch
->intr_ring
.comp_cb_array
=
kzalloc(sizeof(*ch
->intr_ring
.comp_cb_array
) * NUM_COMP_BUFS
, GFP_KERNEL
);
init_ring(&ch
->intr_ring
.ring
, NUM_COMP_BUFS
, dma_ctx
->device_num
);
ch
->intr_ring
.old_tail
= 0;
mi_mic_dma_chan_destroy(struct dma_channel
*ch
, struct mic_dma_ctx_t
*dma_ctx
)
uninit_ring(&ch
->intr_ring
.ring
, dma_ctx
->device_num
);
kfree(ch
->intr_ring
.comp_cb_array
);
uninit_ring(&ch
->poll_ring
, dma_ctx
->device_num
);
open_dma_device(int device_num
, uint8_t *mmio_va_base
, mic_dma_handle_t
* dma_handle
)
if (device_num
>= MAX_BOARD_SUPPORTED
)
mutex_lock(&lock_dma_dev_init
[device_num
]);
if (!mic_dma_context
[device_num
]) {
mic_dma_context
[device_num
] = kzalloc(sizeof(struct mic_dma_ctx_t
), GFP_KERNEL
);
BUG_ON(!mic_dma_context
[device_num
]);
mic_dma_context
[device_num
]->device_num
= device_num
;
result
= mic_dma_lib_init(mmio_va_base
, mic_dma_context
[device_num
]);
atomic_inc(&mic_dma_context
[device_num
]->ref_count
);
*dma_handle
= mic_dma_context
[device_num
];
mutex_unlock(&lock_dma_dev_init
[device_num
]);
EXPORT_SYMBOL(open_dma_device
);
close_dma_device(int device_num
, mic_dma_handle_t
*dma_handle
)
struct mic_dma_ctx_t
*dma_ctx
;
if (device_num
>= MAX_BOARD_SUPPORTED
)
mutex_lock(&lock_dma_dev_init
[device_num
]);
dma_ctx
= (struct mic_dma_ctx_t
*) *dma_handle
;
atomic_read(&dma_ctx
->ref_count
) &&
atomic_dec_and_test(&dma_ctx
->ref_count
)) {
mic_dma_lib_uninit(dma_ctx
);
mic_dma_context
[dma_ctx
->device_num
] = 0;
mutex_unlock(&lock_dma_dev_init
[device_num
]);
EXPORT_SYMBOL(close_dma_device
);
void mi_mic_dma_chan_set_dstat_wb(struct mic_dma_ctx_t
*dma_ctx
,
struct md_mic_dma_chan
*chan
)
if (!chan
->dstat_wb_phys
) {
chan
->dstat_wb_loc
= kzalloc(sizeof(uint32_t), GFP_KERNEL
);
chan
->dstat_wb_phys
= virt_to_phys(chan
->dstat_wb_loc
);
micscif_pci_dev(dma_ctx
->device_num
, &pdev
);
chan
->dstat_wb_phys
= mic_map_single(dma_ctx
->device_num
- 1, pdev
, chan
->dstat_wb_loc
,
BUG_ON(pci_dma_mapping_error(pdev
, chan
->dstat_wb_phys
));
md_mic_dma_chan_set_dstat_wb(&dma_ctx
->dma_dev
, chan
);
md_mic_dma_chan_setup(struct mic_dma_ctx_t
*dma_ctx
, struct dma_channel
*ch
)
md_mic_dma_chan_unmask_intr(&dma_ctx
->dma_dev
, ch
->chan
);
* Disable the channel, update desc ring base and size, write new head
* and then enable the channel.
if (mic_hw_family(ch
->dma_ctx
->device_num
) == FAMILY_KNC
&&
mic_hw_stepping(ch
->dma_ctx
->device_num
) >= KNC_B0_STEP
) {
mi_mic_dma_chan_set_dstat_wb(dma_ctx
, ch
->chan
);
md_mic_dma_chan_set_dcherr_msk(&dma_ctx
->dma_dev
, ch
->chan
, 0);
md_mic_dma_chan_set_desc_ring(&dma_ctx
->dma_dev
, ch
->chan
,
md_mic_dma_chan_unmask_intr(&dma_ctx
->dma_dev
, ch
->chan
);
mic_dma_lib_init(uint8_t *mmio_va_base
, struct mic_dma_ctx_t
*dma_ctx
)
enum md_mic_dma_chan_owner owner
, currentOwner
;
//pr_debug(PR_PREFIX "Initialized the dma mmio va=%p\n", mmio_va_base);
// Using this to check where the DMA lib is at for now.
currentOwner
= mmio_va_base
== 0 ? MIC_DMA_CHAN_MIC_OWNED
: MIC_DMA_CHAN_HOST_OWNED
;
// TODO: multi-card support
md_mic_dma_init(&dma_ctx
->dma_dev
, mmio_va_base
);
for (i
= 0 ; i
< MAX_NUM_DMA_CHAN
; i
++) {
ch
= &dma_ctx
->dma_channels
[i
];
/* Initialize pointer to parent */
owner
= i
> __LAST_HOST_CHAN_NUM
? MIC_DMA_CHAN_MIC_OWNED
: MIC_DMA_CHAN_HOST_OWNED
;
// This has to be done from card side
ch
->chan
= md_mic_dma_request_chan(&dma_ctx
->dma_dev
, owner
);
KASSERT((ch
->chan
!= NULL
), "dummy\n");
ch
->ch_num
= ch
->chan
->ch_num
;
* Host driver would have executed by now and thus setup the
if (ch
->chan
->owner
== MIC_DMA_CHAN_HOST_OWNED
)
md_mic_dma_enable_chan(&dma_ctx
->dma_dev
, i
, true);
atomic_set(&(ch
->flags
), CHAN_INUSE
); // Mark as used by default
if (currentOwner
== owner
) {
alloc_dma_desc_ring_mem(ch
, dma_ctx
);
#ifdef _MIC_SCIF_ // DMA now shares the IRQ handler with other system interrupts
ret_value
= request_irq(i
, dma_interrupt_handler
, IRQF_DISABLED
,
//pr_debug(PR_PREFIX "Interrupt handler ret value for chan %d = %d\n", i, ret_value);
md_mic_dma_chan_setup(dma_ctx
, ch
);
mi_mic_dma_chan_setup(ch
, dma_ctx
);
init_waitqueue_head(&ch
->intr_wq
);
init_waitqueue_head(&ch
->access_wq
);
// Only mark owned channel to be available
atomic_set(&(ch
->flags
), CHAN_AVAILABLE
);
md_mic_dma_print_debug(&dma_ctx
->dma_dev
, ch
->chan
);
/* Initialize last_allocated_dma_channel */
dma_ctx
->last_allocated_dma_channel_num
= -1;
//pr_debug(PR_PREFIX "Initialized the dma channels\n");
mic_dma_proc_init(dma_ctx
);
mic_dma_lib_uninit(struct mic_dma_ctx_t
*dma_ctx
)
mic_dma_proc_uninit(dma_ctx
);
for (i
= 0 ; i
< MAX_NUM_DMA_CHAN
; i
++) {
ch
= &dma_ctx
->dma_channels
[i
];
/* Request the channel but don't free it. Errors are okay */
#ifdef _MIC_SCIF_ // DMA now shares the IRQ handler with other system interrupts
mi_mic_dma_chan_destroy(ch
, dma_ctx
);
micscif_pci_dev(dma_ctx
->device_num
, &pdev
);
mic_unmap_single(dma_ctx
->device_num
- 1, pdev
, ch
->desc_ring_phys
,
(DMA_DESC_RING_SIZE
* sizeof(*ch
->desc_ring
)) + PAGE_SIZE
);
kfree(ch
->desc_ring_bak
);
ch
->desc_ring_bak
= NULL
;
if (mic_hw_family(ch
->dma_ctx
->device_num
) == FAMILY_KNC
&&
mic_hw_stepping(dma_ctx
->device_num
) >= KNC_B0_STEP
) {
mic_unmap_single(dma_ctx
->device_num
- 1, pdev
, ch
->chan
->dstat_wb_phys
,
kfree(ch
->chan
->dstat_wb_loc
);
ch
->chan
->dstat_wb_loc
= NULL
;
ch
->chan
->dstat_wb_phys
= 0;
md_mic_dma_free_chan(&dma_ctx
->dma_dev
, ch
->chan
);
/* Ensure that all waiters for DMA channels time out */
msleep(DMA_TO
/HZ
* 1000);
md_mic_dma_uninit(&dma_ctx
->dma_dev
);
//pr_debug(PR_PREFIX "Uninitialized the dma channels\n");
* reserve_dma_channel - reserve a given dma channel for exclusive use
* @dma_handle - handle to DMA device returned by open_dma_device
* @chan_num - Channel number to be reserved
* @chan - set to point to the dma channel reserved by the call
* Returns < 1 on error (errorno)
* NOTES: Should this function sleep waiting for the lock?
* Maybe there should be a blocking and non-blocking versions of this function
reserve_dma_channel(mic_dma_handle_t dma_handle
, int chan_num
, struct dma_channel
**chan
)
struct mic_dma_ctx_t
*dma_ctx
= (struct mic_dma_ctx_t
*) dma_handle
;
* Do we need to do acquire the lock for statically allocated channels?
* I am assuming we dont have to lock
if (CHAN_AVAILABLE
== atomic_cmpxchg(&(dma_ctx
->dma_channels
[chan_num
].flags
),
CHAN_AVAILABLE
, CHAN_INUSE
)) {
*chan
= &dma_ctx
->dma_channels
[chan_num
];
EXPORT_SYMBOL(reserve_dma_channel
);
* allocate_dma_channel - dynamically allocate a dma channel (for a short while). Will
* search for, choose, and lock down one channel for use by the calling thread.
* @dma_handle - handle to DMA device returned by open_dma_device
* @chan - Returns the dma_channel pointer that was allocated by the call
* NOTE: This function grabs a lock before exiting -- the calling thread MUST NOT
* sleep, and must call free_dma_channel before returning to user-space or switching
* volantarily to another thread. Similarly, this function cannot be called from
* an interrupt context at this time.
* TODO: How do we pick a dma channel?
* For now I am doing it in round robin fashion.
allocate_dma_channel(mic_dma_handle_t dma_handle
, struct dma_channel
**chan
)
struct mic_dma_ctx_t
*dma_ctx
= (struct mic_dma_ctx_t
*) dma_handle
;
j
= dma_ctx
->last_allocated_dma_channel_num
+ 1;
for (i
= 0; i
< MAX_NUM_DMA_CHAN
; i
++, j
++) {
if (CHAN_AVAILABLE
== atomic_cmpxchg(&(dma_ctx
->dma_channels
[j
%
MAX_NUM_DMA_CHAN
].flags
),
CHAN_AVAILABLE
, CHAN_INUSE
)) {
*chan
= &(dma_ctx
->dma_channels
[j
% MAX_NUM_DMA_CHAN
]);
dma_ctx
->last_allocated_dma_channel_num
= j
% MAX_NUM_DMA_CHAN
;
EXPORT_SYMBOL(allocate_dma_channel
);
* request_dma_channel - Request a specific DMA channel.
* @chan - Returns the dma_channel pointer that was requested
* Returns: 0 on success and -ERESTARTSYS if the wait was interrupted
* or -EBUSY if the channel was not available.
* NOTE: This function must call free_dma_channel before returning to
int request_dma_channel(struct dma_channel
*chan
)
ret
= wait_event_interruptible_timeout(chan
->access_wq
,
CHAN_AVAILABLE
== atomic_cmpxchg(&chan
->flags
,
CHAN_AVAILABLE
, CHAN_INUSE
), DMA_TO
);
printk(KERN_ERR
"%s %d TO chan 0x%x\n", __func__
, __LINE__
, chan
->ch_num
);
EXPORT_SYMBOL(request_dma_channel
);
* free_dma_channel - after allocating a channel, used to
* free the channel after DMAs are submitted
* @chan - pointer to the dma_channel struct that was allocated
* Returns 0 on success, < 1 on error (errorno)
* NOTE: This function must be called after all do_dma calls are finished,
* but can be called before the DMAs actually complete (as long as the comp_cb()
* handler in do_dma don't refer to the dma_channel struct). If called with a
* dynamically allocated dma_chan, the caller must be the thread that called
* allocate_dma_chan. When operating on a dynamic channel, free unlocks the
* mutex locked in allocate. Statically allocated channels cannot be freed,
* and calling this function with that type of channel will return an error.
free_dma_channel(struct dma_channel
*chan
)
* Why can't we use this function with channels that were statically allocated??
atomic_cmpxchg(&chan
->flags
, CHAN_INUSE
, CHAN_AVAILABLE
));
wake_up(&chan
->access_wq
);
EXPORT_SYMBOL(free_dma_channel
);
static __always_inline
uint32_t
get_dma_tail_pointer(struct dma_channel
*chan
)
struct mic_dma_device
*dma_dev
;
dma_dev
= &chan
->dma_ctx
->dma_dev
;
return md_mic_dma_chan_read_tail(dma_dev
, chan
->chan
);
* Return -1 in case of error
program_memcpy_descriptors(struct dma_channel
*chan
, uint64_t src
, uint64_t dst
, size_t len
)
size_t current_transfer_len
;
unsigned long ts
= jiffies
;
if (mic_hw_family(chan
->dma_ctx
->device_num
) == FAMILY_KNC
) {
if (mic_hw_stepping(chan
->dma_ctx
->device_num
) == KNC_A_STEP
)
current_transfer_len
= (len
> MAX_DMA_XFER_SIZE
) ?
while (!md_avail_desc_ring_space(&chan
->dma_ctx
->dma_dev
, is_astep
, chan
->chan
,
(uint32_t)chan
->next_write_index
, 1)) {
if (time_after(jiffies
,ts
+ DMA_TO
)) {
printk(KERN_ERR
"%s %d TO chan 0x%x\n", __func__
, __LINE__
, chan
->ch_num
);
//pr_debug("src_phys=0x%llx, dst_phys=0x%llx, size=0x%zx\n", src_phys_addr, dst_phys_addr, current_transfer_len);
md_mic_dma_memcpy_desc(&chan
->desc_ring
[chan
->next_write_index
],
src
, dst
, current_transfer_len
);
chan
->next_write_index
= incr_rb_index((int)chan
->next_write_index
,
chan
->chan
->num_desc_in_ring
);
len
-= current_transfer_len
;
dst
= dst
+ current_transfer_len
;
src
= src
+ current_transfer_len
;
* do_dma - main dma function: perform a dma memcpy, len bytes from src to dst
* @chan - DMA channel to use for the transfer. The channel can be allocated
* dynamically by calling allocate_dma_chan, or statically by
* reserve_dma_chan. Using a channel not allocated in this way will
* result in undefined behavior.
* @flags - ATOMIC, called from an interrupt context (no blocking)
* @src - src physical address
* @dst - dst physical address
* @len - Length of the dma
* @comp_cb - When the DMA is complete, the struct's function will be called. NOTE!
* comp_cb(cb_cookie) is called from an interrupt context, so the
* function must not sleep or block.
* TODO: Figure out proper value instead of -2
* Return = -2 copy was done successfully, no need to wait
* Return >= 0: DMA has been queued. Return value can be polled on for completion
* if DO_DMA_POLLING was sent in flags
* (poll cookie). An example (simplified w/ no error handling).
* int cookie = do_dma(...);
* while (poll_dma_completion(cookie) == 0);
* printf("DMA now complete\n");
do_dma(struct dma_channel
*chan
, int flags
, uint64_t src
,
uint64_t dst
, size_t len
, struct dma_completion_cb
*comp_cb
)
* Do we need to assert the ownership of channel??
int poll_ring_index
= -1;
int intr_ring_index
= -1;
uint32_t num_status_desc
= 0;
unsigned long ts
= jiffies
;
if (flags
& DO_DMA_INTR
&& !comp_cb
)
if (!verify_next_write_index(chan
))
//pr_debug(PR_PREFIX "Current transfer src = 0x%llx,dst = 0x%llx, len = 0x%zx\n", src, dst, len);
if (flags
& DO_DMA_INTR
) {
err
= wait_event_interruptible_timeout(chan
->intr_wq
,
(-1 != (intr_ring_index
= allocate_buffer(&chan
->intr_ring
.ring
))),
printk(KERN_ERR
"%s %d TO chan 0x%x\n", __func__
, __LINE__
, chan
->ch_num
);
chan
->intr_ring
.comp_cb_array
[intr_ring_index
] = comp_cb
;
//pr_debug(PR_PREFIX "INTR intr_ring_index=%d, chan_num=%lx\n", intr_ring_index, (chan - dma_channels));
if (flags
& DO_DMA_POLLING
) {
poll_ring_index
= allocate_buffer(&chan
->poll_ring
);
if (-1 == poll_ring_index
)
//pr_debug(PR_PREFIX "polling poll_ring_index=%d\n", poll_ring_index);
if (len
&& -ENOMEM
== program_memcpy_descriptors(chan
, src
, dst
, len
)) {
//pr_debug(PR_PREFIX "ERROR: do_dma: No available space from program_memcpy_descriptors\n");
if (mic_hw_family(chan
->dma_ctx
->device_num
) == FAMILY_KNC
) {
if (mic_hw_stepping(chan
->dma_ctx
->device_num
) == KNC_A_STEP
)
while (num_status_desc
&& num_status_desc
> md_avail_desc_ring_space(&chan
->dma_ctx
->dma_dev
,
is_astep
, chan
->chan
, (uint32_t)chan
->next_write_index
, num_status_desc
)) {
if (time_after(jiffies
,ts
+ DMA_TO
)) {
printk(KERN_ERR
"%s %d TO chan 0x%x\n", __func__
, __LINE__
, chan
->ch_num
);
//pr_debug(PR_PREFIX "ERROR: do_dma: No available space from md_avail_desc_ring_space\n");
if (flags
& DO_DMA_POLLING
) {
incr_head(&chan
->poll_ring
);
md_mic_dma_prep_status_desc(&chan
->desc_ring
[chan
->next_write_index
],
chan
->poll_ring
.tail_phys
,
chan
->next_write_index
= incr_rb_index((int)chan
->next_write_index
,
chan
->chan
->num_desc_in_ring
);
if (flags
& DO_DMA_INTR
) {
incr_head(&chan
->intr_ring
.ring
);
md_mic_dma_prep_status_desc(&chan
->desc_ring
[chan
->next_write_index
],
chan
->intr_ring
.ring
.tail_phys
,
chan
->next_write_index
= incr_rb_index((int)chan
->next_write_index
,
chan
->chan
->num_desc_in_ring
);
md_mic_dma_prep_status_desc(&chan
->desc_ring
[chan
->next_write_index
],
chan
->intr_ring
.ring
.tail_phys
,
chan
->next_write_index
= incr_rb_index((int)chan
->next_write_index
,
chan
->chan
->num_desc_in_ring
);
* Maybe it is better if we update the head pointer for every descriptor??
md_mic_dma_chan_write_head(&chan
->dma_ctx
->dma_dev
, chan
->chan
, (uint32_t)chan
->next_write_index
);
//pr_debug(PR_PREFIX "in HW chan->next_write_index=%lld\n", chan->next_write_index);
if (DO_DMA_POLLING
& flags
)
* poll_dma_completion - check if a DMA is complete
* @poll_cookie - value returned from do_dma
* Note: This is mostly useful after calling do_dma with a NULL comp_cb parameter, as
* it will allow the caller to wait for DMA completion.
poll_dma_completion(int poll_cookie
, struct dma_channel
*chan
)
* In case of interrupts the ISR runs and reads the value
* of the tail location. If we are polling then we need
* to read the value of the tail location before checking
* if the entry is processed.
chan
->poll_ring
.tail
= read_tail(&chan
->poll_ring
);
return is_entry_processed(&chan
->poll_ring
, poll_cookie
);
EXPORT_SYMBOL(poll_dma_completion
);
* do_status_update: Update physical address location with the value provided.
* Ensures all previous DMA descriptors submitted on this DMA
* @chan - DMA channel to use for the transfer. The channel can be allocated
* dynamically by calling allocate_dma_channel, or statically by
* reserve_dma_channel. Using a channel not allocated in this way will
* result in undefined behavior.
* @phys - physical address
* @value - Value to be programmed
int do_status_update(struct dma_channel
*chan
, uint64_t phys
, uint64_t value
)
unsigned long ts
= jiffies
;
if (!verify_next_write_index(chan
))
if (mic_hw_family(chan
->dma_ctx
->device_num
) == FAMILY_KNC
) {
if (mic_hw_stepping(chan
->dma_ctx
->device_num
) == KNC_A_STEP
)
* Do we need to assert the ownership of channel??
while (!md_avail_desc_ring_space(&chan
->dma_ctx
->dma_dev
,
is_astep
, chan
->chan
, (uint32_t) chan
->next_write_index
, 1)) {
if (time_after(jiffies
,ts
+ DMA_TO
)) {
printk(KERN_ERR
"%s %d TO chan 0x%x\n", __func__
, __LINE__
, chan
->ch_num
);
md_mic_dma_prep_status_desc(&chan
->desc_ring
[chan
->next_write_index
],
chan
->next_write_index
= incr_rb_index((int)chan
->next_write_index
,
chan
->chan
->num_desc_in_ring
);
md_mic_dma_chan_write_head(&chan
->dma_ctx
->dma_dev
,
chan
->chan
, (uint32_t)chan
->next_write_index
);
EXPORT_SYMBOL(do_status_update
);
* get_dma_mark: Obtain current value of DMA mark
* @chan - DMA channel to use for the transfer. The channel can be allocated
* dynamically by calling allocate_dma_channel, or statically by
* reserve_dma_channel. Using a channel not allocated in this way will
* result in undefined behavior.
int get_dma_mark(struct dma_channel
*chan
)
return chan
->intr_ring
.ring
.head
;
EXPORT_SYMBOL(get_dma_mark
);
* program_dma_mark: Increment the current value of the DMA mark for a DMA channel
* and program an interrupt status update descriptor which ensures that all DMA
* descriptors programmed uptil this point in time are completed.
* @chan - DMA channel to use for the transfer. The channel can be allocated
* dynamically by calling allocate_dma_channel, or statically by
* reserve_dma_channel. Using a channel not allocated in this way will
* result in undefined behavior.
int program_dma_mark(struct dma_channel
*chan
)
* Do we need to assert the ownership of channel??
unsigned long ts
= jiffies
;
uint32_t num_status_desc
= 1;
if (!verify_next_write_index(chan
))
if (mic_hw_family(chan
->dma_ctx
->device_num
) == FAMILY_KNC
) {
if (mic_hw_stepping(chan
->dma_ctx
->device_num
) == KNC_A_STEP
)
err
= wait_event_interruptible_timeout(chan
->intr_wq
,
(-1 != (intr_ring_index
= allocate_buffer(&chan
->intr_ring
.ring
))),
while (num_status_desc
> md_avail_desc_ring_space(&chan
->dma_ctx
->dma_dev
,
is_astep
, chan
->chan
, (uint32_t)chan
->next_write_index
, num_status_desc
)) {
if (time_after(jiffies
,ts
+ DMA_TO
)) {
printk(KERN_ERR
"%s %d TO chan 0x%x\n", __func__
, __LINE__
, chan
->ch_num
);
chan
->intr_ring
.comp_cb_array
[intr_ring_index
] = NULL
;
incr_head(&chan
->intr_ring
.ring
);
md_mic_dma_prep_status_desc(&chan
->desc_ring
[chan
->next_write_index
],
chan
->intr_ring
.ring
.tail_phys
,
chan
->next_write_index
= incr_rb_index((int)chan
->next_write_index
,
chan
->chan
->num_desc_in_ring
);
md_mic_dma_prep_status_desc(&chan
->desc_ring
[chan
->next_write_index
],
chan
->intr_ring
.ring
.tail_phys
,
chan
->next_write_index
= incr_rb_index((int)chan
->next_write_index
,
chan
->chan
->num_desc_in_ring
);
md_mic_dma_chan_write_head(&chan
->dma_ctx
->dma_dev
, chan
->chan
, (uint32_t)chan
->next_write_index
);
EXPORT_SYMBOL(program_dma_mark
);
* is_current_dma_mark: Check if the dma mark provided is the current DMA mark.
* Return true on success and false on failure.
bool is_current_dma_mark(struct dma_channel
*chan
, int mark
)
return (get_dma_mark(chan
) == mark
);
EXPORT_SYMBOL(is_current_dma_mark
);
* is_dma_mark_processed: Check if the dma mark provided has been processed.
* Return true on success and false on failure.
bool is_dma_mark_processed(struct dma_channel
*chan
, int mark
)
return is_entry_processed(&chan
->intr_ring
.ring
, mark
);
EXPORT_SYMBOL(is_dma_mark_processed
);
* @is_interruptible - Use wait_event_interruptible() or not.
* Wait for the dma mark to complete.
* Return 0 on success and appropriate error value on error.
int dma_mark_wait(struct dma_channel
*chan
, int mark
, bool is_interruptible
)
uint32_t prev_tail
= 0, new_tail
;
err
= wait_event_interruptible_timeout(
is_dma_mark_processed(chan
, mark
),
err
= wait_event_timeout(chan
->intr_wq
,
is_dma_mark_processed(chan
, mark
), DMA_TO
);
if (!err
) { // 0 is timeout
new_tail
= get_dma_tail_pointer(chan
);
if ((count
<= DMA_FENCE_TIMEOUT_CNT
) &&
(!count
|| new_tail
!= prev_tail
)) { // For performance, prev_tail is not read at the begining
pr_debug("DMA fence wating is still ongoing, waiting for %d seconds\n", DMA_TO
/HZ
*count
);
printk(KERN_ERR
"%s %d TO chan 0x%x\n", __func__
, __LINE__
, chan
->ch_num
);
EXPORT_SYMBOL(dma_mark_wait
);
* drain_dma_poll - Drain all outstanding DMA operations for a particular
* DMA channel via polling.
* Return 0 on success and -errno on error.
int drain_dma_poll(struct dma_channel
*chan
)
uint32_t prev_tail
= 0, new_tail
, count
= 0;
if ((err
= request_dma_channel(chan
)))
if ((cookie
= do_dma(chan
,
DO_DMA_POLLING
, 0, 0, 0, NULL
)) < 0) {
while (1 != poll_dma_completion(cookie
, chan
)) {
if (time_after(jiffies
,ts
+ DMA_TO
)) {
new_tail
= get_dma_tail_pointer(chan
);
if ((!count
|| new_tail
!= prev_tail
) && (count
<= DMA_FENCE_TIMEOUT_CNT
)) {
pr_debug("polling DMA is still ongoing, wating for %d seconds\n", DMA_TO
/HZ
* count
);
printk(KERN_ERR
"%s %d err %d\n", __func__
, __LINE__
, err
);
EXPORT_SYMBOL(drain_dma_poll
);
* drain_dma_intr - Drain all outstanding DMA operations for a particular
* DMA channel via interrupt based blocking wait.
* Return 0 on success and -errno on error.
int drain_dma_intr(struct dma_channel
*chan
)
if ((err
= request_dma_channel(chan
)))
if ((cookie
= program_dma_mark(chan
)) < 0) {
err
= dma_mark_wait(chan
, cookie
, false);
printk(KERN_ERR
"%s %d err %d\n", __func__
, __LINE__
, err
);
EXPORT_SYMBOL(drain_dma_intr
);
* drain_dma_global - Drain all outstanding DMA operations for
* all online DMA channel.
int drain_dma_global(mic_dma_handle_t dma_handle
)
struct dma_channel
*chan
;
struct mic_dma_ctx_t
*dma_ctx
= (struct mic_dma_ctx_t
*)dma_handle
;
for (i
= 0 ; i
< MAX_NUM_DMA_CHAN
; i
++) {
chan
= &dma_ctx
->dma_channels
[i
];
if (chan
->desc_ring
== NULL
)
if ((err
= drain_dma_intr(chan
)))
EXPORT_SYMBOL(drain_dma_global
);
* dma_suspend: DMA tasks before transition to low power state.
* @dma_handle: Handle for a DMA driver context.
* Perform the following tasks before the device transitions
* 1) Store away the DMA descriptor ring physical address base for
* all DMA channels (both host/uOS owned) since the value would be
* required to reinitialize the DMA channels upon transition from
* low power to active state.
* Notes: Invoked only on MIC.
void dma_suspend(mic_dma_handle_t dma_handle
)
struct mic_dma_ctx_t
*dma_ctx
= (struct mic_dma_ctx_t
*)dma_handle
;
struct mic_dma_device
*dma_dev
= &dma_ctx
->dma_dev
;
for (i
= 0; i
< MAX_NUM_DMA_CHAN
; i
++) {
ch
= &dma_ctx
->dma_channels
[i
];
md_mic_dma_chan_get_desc_ring_phys(dma_dev
, ch
->chan
);
ch
->chan
->dstat_wb_phys
=
md_mic_dma_chan_get_dstatwb_phys(dma_dev
, ch
->chan
);
EXPORT_SYMBOL(dma_suspend
);
* dma_resume: DMA tasks after wake up from low power state.
* @dma_handle: Handle for a DMA driver context.
* Performs the following tasks before the device transitions
* from a low power state to active state:
* 1) As a test, reset the value in DMA configuration register.
* 2) Reset the next_write_index for the DMA descriptor ring to 0
* since the DMA channel will be reset shortly.
* 3) Reinitialize the DMA MD layer for the channel.
* Notes: Invoked only on MIC.
void dma_resume(mic_dma_handle_t dma_handle
)
struct mic_dma_ctx_t
*dma_ctx
= (struct mic_dma_ctx_t
*)dma_handle
;
struct mic_dma_device
*dma_dev
= &dma_ctx
->dma_dev
;
/* TODO: Remove test write to SBOX_DCR */
mic_sbox_write_mmio(dma_dev
->mm_sbox
, SBOX_DCR
, 0);
for (i
= 0; i
< MAX_NUM_DMA_CHAN
; i
++) {
ch
= &dma_ctx
->dma_channels
[i
];
ch
->next_write_index
= 0;
md_mic_dma_chan_init_attr(dma_dev
, ch
->chan
);
md_mic_dma_chan_setup(dma_ctx
, ch
);
EXPORT_SYMBOL(dma_resume
);
* dma_prep_suspend: DMA tasks required on host before a device can transition
* @dma_handle: Handle for a DMA driver context.
* Performs the following tasks on the host before the device can be allowed
* to transiti to a low power state.
* 1) Reset the next_Write_index for the DMA descriptor ring to 0
* since the DMA channel will be reset shortly. This is required primarily
* for Host owned DMA channels since MIC does not have access to this
void dma_prep_suspend(mic_dma_handle_t dma_handle
)
struct mic_dma_ctx_t
*dma_ctx
= (struct mic_dma_ctx_t
*)dma_handle
;
for (i
= 0; i
< MAX_NUM_DMA_CHAN
; i
++) {
ch
= &dma_ctx
->dma_channels
[i
];
ch
->next_write_index
= 0;
EXPORT_SYMBOL(dma_prep_suspend
);
#ifdef CONFIG_PAGE_CACHE_DMA
static const struct dma_operations dma_operations_fast_copy
= {
.poll_dma_completion
= poll_dma_completion
,
.free_dma_channel
= free_dma_channel
,
.open_dma_device
= open_dma_device
,
.close_dma_device
= close_dma_device
,
.allocate_dma_channel
= allocate_dma_channel
,
.program_descriptors
= program_memcpy_descriptors
,
.do_dma_polling
= DO_DMA_POLLING
,
static const struct file_dma fdma_callback
= {
.dmaops
= &dma_operations_fast_copy
,
for (i
= 0; i
< MAX_BOARD_SUPPORTED
; i
++)
mutex_init (&lock_dma_dev_init
[i
]);
#ifdef CONFIG_PAGE_CACHE_DMA
register_dma_for_fast_copy(&fdma_callback
);
static void mic_dma_uninit(void)
#ifdef CONFIG_PAGE_CACHE_DMA
unregister_dma_for_fast_copy();
module_init(mic_dma_init
);
module_exit(mic_dma_uninit
);
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0))
mic_dma_proc_ring_show(struct seq_file
*m
, void *data
)
struct mic_dma_ctx_t
*dma_ctx
= m
->private;
mic_ctx_t
*mic_ctx
= get_per_dev_ctx(dma_ctx
->device_num
- 1);
struct compl_buf_ring
*ring
;
if ((err
= micpm_get_reference(mic_ctx
, true))) {
printk(KERN_ERR
"%s %d: unable to get micpm reference: %d\n",
__func__
, __LINE__
, err
);
seq_printf(m
, "Intr rings\n");
seq_printf(m
, "%-10s%-12s%-12s%-12s%-25s%-18s%-25s\n",
"Chan", "Head", "Tail", "Size", "Tail loc", "Actual tail", "In Use");
for (i
= first_dma_chan(); i
<= last_dma_chan(); i
++) {
ring
= &dma_ctx
->dma_channels
[i
].intr_ring
.ring
;
seq_printf(m
, "%-#10x%-#12x%-#12x%-#12x%-#25llx%-#18x%-#18x\n",
i
, ring
->head
, ring
->tail
, ring
->size
,
ring
->tail_location
, *(int*)ring
->tail_location
,
atomic_read(&dma_ctx
->dma_channels
[i
].flags
));
seq_printf(m
, "Poll rings\n");
seq_printf(m
, "%-10s%-12s%-12s%-12s%-25s%-18s\n",
"Chan", "Head", "Tail", "Size", "Tail loc", "Actual tail");
for (i
= first_dma_chan(); i
<= last_dma_chan(); i
++) {
ring
= &dma_ctx
->dma_channels
[i
].poll_ring
;
seq_printf(m
, "%-#10x%-#12x%-#12x%-#12x%-#25llx%-#18x\n",
i
, ring
->head
, ring
->tail
, ring
->size
,
ring
->tail_location
, *(int*)ring
->tail_location
);
seq_printf(m
, "Next_Write_Index\n");
seq_printf(m
, "%-10s%-12s\n", "Chan", "Next_Write_Index");
for (i
= 0; i
< MAX_NUM_DMA_CHAN
; i
++) {
seq_printf(m
, "%-#10x%-#12llx\n",
i
, dma_ctx
->dma_channels
[i
].next_write_index
);
micpm_put_reference(mic_ctx
);
mic_dma_proc_ring_open(struct inode
*inode
, struct file
*file
)
return single_open(file
, mic_dma_proc_ring_show
, PDE_DATA(inode
));
mic_dma_proc_reg_show(struct seq_file
*m
, void *data
)
int i
, j
, chan_num
, size
, dtpr
, err
;
struct mic_dma_ctx_t
*dma_ctx
= m
->private;
mic_ctx_t
*mic_ctx
= get_per_dev_ctx(dma_ctx
->device_num
- 1);
struct mic_dma_device
*dma_dev
= &dma_ctx
->dma_dev
;
struct dma_channel
*curr_chan
;
union md_mic_dma_desc desc
;
if ((err
= micpm_get_reference(mic_ctx
, true))) {
printk(KERN_ERR
"%s %d: unable to get micpm reference: %d\n",
__func__
, __LINE__
, err
);
seq_printf(m
, "========================================"
"=======================================\n");
seq_printf(m
, "SBOX_DCR: %#x\n",
mic_sbox_read_mmio(dma_dev
->mm_sbox
, SBOX_DCR
));
seq_printf(m
, "DMA Channel Registers\n");
seq_printf(m
, "========================================"
"=======================================\n");
seq_printf(m
, "%-10s| %-10s %-10s %-10s %-10s %-10s %-10s"
" %-10s %-11s %-14s %-10s"
"\n", "Channel", "DCAR", "DTPR", "DHPR",
"DSTATWB_LO", "DSTATWB_HI", "DSTAT_CHERR", "DSTAT_CHERRMSK",
seq_printf(m
, "========================================"
"=======================================\n");
for (i
= 0; i
< MAX_NUM_DMA_CHAN
; i
++) {
for (i
= first_dma_chan(); i
<= last_dma_chan(); i
++) {
curr_chan
= &dma_ctx
->dma_channels
[i
];
chan_num
= curr_chan
->ch_num
;
seq_printf(m
, "%-10i| %-#10x %-#10x %-#10x %-#10x"
" %-#10x %-#11x %-#10x %-#14x"
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DCAR
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DTPR
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DHPR
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DRAR_HI
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DRAR_LO
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DSTATWB_LO
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DSTATWB_HI
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DCHERR
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DCHERRMSK
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DSTAT
));
seq_printf(m
, "\nDMA Channel Descriptor Rings\n");
seq_printf(m
, "========================================"
"=======================================\n");
for (i
= first_dma_chan(); i
<= last_dma_chan(); i
++) {
curr_chan
= &dma_ctx
->dma_channels
[i
];
chan_num
= curr_chan
->ch_num
;
dtpr
= md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DTPR
);
seq_printf(m
, "Channel %i: [", chan_num
);
size
= ((int) md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DHPR
)
- dtpr
) % curr_chan
->chan
->num_desc_in_ring
;
* In KNC B0, empty condition is tail = head -1
if (mic_hw_family(dma_ctx
->device_num
) == FAMILY_KNC
&&
mic_hw_stepping(dma_ctx
->device_num
) >= KNC_B0_STEP
)
for (j
= 0; j
< size
; j
++) {
desc
= curr_chan
->desc_ring
[(j
+dtpr
) %
curr_chan
->chan
->num_desc_in_ring
];
switch (desc
.desc
.nop
.type
){
seq_printf(m
," {Type: NOP, 0x%#llx"
" %#llx} ", desc
.qwords
.qw0
,
seq_printf(m
," {Type: MEMCOPY, SAP:"
" 0x%#llx, DAP: %#llx, length: %#llx} ",
(uint64_t) desc
.desc
.memcopy
.sap
,
(uint64_t) desc
.desc
.memcopy
.dap
,
(uint64_t) desc
.desc
.memcopy
.length
);
seq_printf(m
," {Type: STATUS, data:"
" 0x%#llx, DAP: %#llx, intr: %lli} ",
(uint64_t) desc
.desc
.status
.data
,
(uint64_t) desc
.desc
.status
.dap
,
(uint64_t) desc
.desc
.status
.intr
);
seq_printf(m
," {Type: GENERAL, "
"DAP: %#llx, dword: %#llx} ",
(uint64_t) desc
.desc
.general
.dap
,
(uint64_t) desc
.desc
.general
.data
);
seq_printf(m
," {Type: KEYNONCECNT, sel: "
"%lli, h: %lli, index: %lli, cs: %lli,"
(uint64_t) desc
.desc
.keynoncecnt
.sel
,
(uint64_t) desc
.desc
.keynoncecnt
.h
,
(uint64_t) desc
.desc
.keynoncecnt
.index
,
(uint64_t) desc
.desc
.keynoncecnt
.cs
,
(uint64_t) desc
.desc
.keynoncecnt
.data
);
seq_printf(m
," {Type: KEY, dest_ind"
"ex: %lli, ski: %lli, skap: %#llx ",
(uint64_t) desc
.desc
.key
.di
,
(uint64_t) desc
.desc
.key
.ski
,
(uint64_t) desc
.desc
.key
.skap
);
seq_printf(m
," {Uknown Type=%lli ,"
"%#llx %#llx} ",(uint64_t) desc
.desc
.nop
.type
,
(uint64_t) desc
.qwords
.qw0
,
(uint64_t) desc
.qwords
.qw1
);
if (mic_hw_family(dma_ctx
->device_num
) == FAMILY_KNC
&&
mic_hw_stepping(dma_ctx
->device_num
) >= KNC_B0_STEP
&&
curr_chan
->chan
->dstat_wb_loc
)
seq_printf(m
, "DSTAT_WB = 0x%x\n",
*((uint32_t*)curr_chan
->chan
->dstat_wb_loc
));
micpm_put_reference(mic_ctx
);
mic_dma_proc_reg_open(struct inode
*inode
, struct file
*file
)
return single_open(file
, mic_dma_proc_reg_show
, PDE_DATA(inode
));
struct file_operations micdma_ring_fops
= {
.open
= mic_dma_proc_ring_open
,
.release
= single_release
,
struct file_operations micdma_reg_fops
= {
.open
= mic_dma_proc_reg_open
,
.release
= single_release
,
mic_dma_proc_init(struct mic_dma_ctx_t
*dma_ctx
)
snprintf(name
, 63, "%s%d", proc_dma_ring
, dma_ctx
->device_num
);
if (!proc_create_data(name
, S_IFREG
| S_IRUGO
, NULL
, &micdma_ring_fops
, dma_ctx
))
printk("micdma: unable to register /proc/%s\n", name
);
snprintf(name
, 63, "%s%d", proc_dma_reg
, dma_ctx
->device_num
);
if (!proc_create_data(name
, S_IFREG
| S_IRUGO
, NULL
, &micdma_reg_fops
, dma_ctx
))
printk("micdma: unable to register /proc/%s\n", name
);
mic_dma_proc_read_fn(char *buf
, char **start
, off_t offset
, int count
, int *eof
, void *data
)
struct mic_dma_ctx_t
*dma_ctx
= data
;
struct compl_buf_ring
*ring
;
len
+= sprintf(buf
+ len
, "Intr rings\n");
len
+= sprintf(buf
+ len
, "%-10s%-12s%-12s%-12s%-25s%-18s%-25s\n",
"Chan", "Head", "Tail", "Size", "Tail loc", "Actual tail", "In Use");
for (i
= first_dma_chan(); i
<= last_dma_chan(); i
++) {
ring
= &dma_ctx
->dma_channels
[i
].intr_ring
.ring
;
len
+= sprintf(buf
+ len
, "%-#10x%-#12x%-#12x%-#12x%-#25llx%-#18x%-#18x\n",
i
, ring
->head
, ring
->tail
, ring
->size
,
ring
->tail_location
, *(int*)ring
->tail_location
,
atomic_read(&dma_ctx
->dma_channels
[i
].flags
));
len
+= sprintf(buf
+ len
, "Poll rings\n");
len
+= sprintf(buf
+ len
, "%-10s%-12s%-12s%-12s%-25s%-18s\n",
"Chan", "Head", "Tail", "Size", "Tail loc", "Actual tail");
for (i
= first_dma_chan(); i
<= last_dma_chan(); i
++) {
ring
= &dma_ctx
->dma_channels
[i
].poll_ring
;
len
+= sprintf(buf
+ len
, "%-#10x%-#12x%-#12x%-#12x%-#25llx%-#18x\n",
i
, ring
->head
, ring
->tail
, ring
->size
,
ring
->tail_location
, *(int*)ring
->tail_location
);
len
+= sprintf(buf
+ len
, "Next_Write_Index\n");
len
+= sprintf(buf
+ len
, "%-10s%-12s\n", "Chan", "Next_Write_Index");
for (i
= 0; i
< MAX_NUM_DMA_CHAN
; i
++) {
len
+= sprintf(buf
+ len
, "%-#10x%-#12llx\n",
i
, dma_ctx
->dma_channels
[i
].next_write_index
);
mic_dma_proc_read_registers_fn(char *buf
, char **start
, off_t offset
, int count
,
int i
, j
, chan_num
, size
, dtpr
, len
= 0;
struct mic_dma_ctx_t
*dma_ctx
= data
;
struct mic_dma_device
*dma_dev
= &dma_ctx
->dma_dev
;
struct dma_channel
*curr_chan
;
union md_mic_dma_desc desc
;
len
+= sprintf(buf
+ len
, "========================================"
"=======================================\n");
len
+= sprintf(buf
+ len
, "SBOX_DCR: %#x\n",
mic_sbox_read_mmio(dma_dev
->mm_sbox
, SBOX_DCR
));
len
+= sprintf(buf
+ len
, "DMA Channel Registers\n");
len
+= sprintf(buf
+ len
, "========================================"
"=======================================\n");
len
+= sprintf(buf
+ len
, "%-10s| %-10s %-10s %-10s %-10s %-10s %-10s"
" %-10s %-11s %-14s %-10s"
"\n", "Channel", "DCAR", "DTPR", "DHPR",
"DSTATWB_LO", "DSTATWB_HI", "DSTAT_CHERR", "DSTAT_CHERRMSK",
len
+= sprintf(buf
+ len
, "========================================"
"=======================================\n");
for (i
= 0; i
< MAX_NUM_DMA_CHAN
; i
++) {
for (i
= first_dma_chan(); i
<= last_dma_chan(); i
++) {
curr_chan
= &dma_ctx
->dma_channels
[i
];
chan_num
= curr_chan
->ch_num
;
len
+= sprintf(buf
+ len
, "%-10i| %-#10x %-#10x %-#10x %-#10x"
" %-#10x %-#11x %-#10x %-#14x"
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DCAR
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DTPR
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DHPR
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DRAR_HI
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DRAR_LO
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DSTATWB_LO
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DSTATWB_HI
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DCHERR
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DCHERRMSK
),
md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DSTAT
));
len
+= sprintf(buf
+ len
, "\nDMA Channel Descriptor Rings\n");
len
+= sprintf(buf
+ len
, "========================================"
"=======================================\n");
for (i
= first_dma_chan(); i
<= last_dma_chan(); i
++) {
curr_chan
= &dma_ctx
->dma_channels
[i
];
chan_num
= curr_chan
->ch_num
;
dtpr
= md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DTPR
);
len
+= sprintf(buf
+ len
, "Channel %i: [", chan_num
);
size
= ((int) md_mic_dma_read_mmio(dma_dev
, chan_num
, REG_DHPR
)
- dtpr
) % curr_chan
->chan
->num_desc_in_ring
;
* In KNC B0, empty condition is tail = head -1
if (mic_hw_family(dma_ctx
->device_num
) == FAMILY_KNC
&&
mic_hw_stepping(dma_ctx
->device_num
) >= KNC_B0_STEP
)
for (j
= 0; j
< size
; j
++) {
desc
= curr_chan
->desc_ring
[(j
+dtpr
) %
curr_chan
->chan
->num_desc_in_ring
];
switch (desc
.desc
.nop
.type
){
len
+= sprintf(buf
+ len
," {Type: NOP, 0x%#llx"
" %#llx} ", desc
.qwords
.qw0
,
len
+= sprintf(buf
+ len
," {Type: MEMCOPY, SAP:"
" 0x%#llx, DAP: %#llx, length: %#llx} ",
(uint64_t) desc
.desc
.memcopy
.sap
,
(uint64_t) desc
.desc
.memcopy
.dap
,
(uint64_t) desc
.desc
.memcopy
.length
);
len
+= sprintf(buf
+ len
," {Type: STATUS, data:"
" 0x%#llx, DAP: %#llx, intr: %lli} ",
(uint64_t) desc
.desc
.status
.data
,
(uint64_t) desc
.desc
.status
.dap
,
(uint64_t) desc
.desc
.status
.intr
);
len
+= sprintf(buf
+ len
," {Type: GENERAL, "
"DAP: %#llx, dword: %#llx} ",
(uint64_t) desc
.desc
.general
.dap
,
(uint64_t) desc
.desc
.general
.data
);
len
+= sprintf(buf
+ len
," {Type: KEYNONCECNT, sel: "
"%lli, h: %lli, index: %lli, cs: %lli,"
(uint64_t) desc
.desc
.keynoncecnt
.sel
,
(uint64_t) desc
.desc
.keynoncecnt
.h
,
(uint64_t) desc
.desc
.keynoncecnt
.index
,
(uint64_t) desc
.desc
.keynoncecnt
.cs
,
(uint64_t) desc
.desc
.keynoncecnt
.data
);
len
+= sprintf(buf
+ len
," {Type: KEY, dest_ind"
"ex: %lli, ski: %lli, skap: %#llx ",
(uint64_t) desc
.desc
.key
.di
,
(uint64_t) desc
.desc
.key
.ski
,
(uint64_t) desc
.desc
.key
.skap
);
len
+= sprintf(buf
+ len
," {Uknown Type=%lli ,"
"%#llx %#llx} ",(uint64_t) desc
.desc
.nop
.type
,
(uint64_t) desc
.qwords
.qw0
,
(uint64_t) desc
.qwords
.qw1
);
len
+= sprintf(buf
+ len
, "]\n");
if (mic_hw_family(dma_ctx
->device_num
) == FAMILY_KNC
&&
mic_hw_stepping(dma_ctx
->device_num
) >= KNC_B0_STEP
&&
curr_chan
->chan
->dstat_wb_loc
)
len
+= sprintf(buf
+ len
, "DSTAT_WB = 0x%x\n",
*((uint32_t*)curr_chan
->chan
->dstat_wb_loc
));
mic_dma_proc_init(struct mic_dma_ctx_t
*dma_ctx
)
struct proc_dir_entry
*dma_proc
;
snprintf(name
, 63, "%s%d", proc_dma_ring
, dma_ctx
->device_num
);
if ((dma_proc
= create_proc_entry(name
, S_IFREG
| S_IRUGO
, NULL
)) != NULL
) {
dma_proc
->read_proc
= mic_dma_proc_read_fn
;
dma_proc
->data
= dma_ctx
;
snprintf(name
, 63, "%s%d", proc_dma_reg
, dma_ctx
->device_num
);
if ((dma_proc
= create_proc_entry(name
, S_IFREG
| S_IRUGO
, NULL
)) != NULL
) {
dma_proc
->read_proc
= mic_dma_proc_read_registers_fn
;
dma_proc
->data
= dma_ctx
;
mic_dma_proc_uninit(struct mic_dma_ctx_t
*dma_ctx
)
snprintf(name
, 63, "%s%d", proc_dma_reg
, dma_ctx
->device_num
);
remove_proc_entry(name
, NULL
);
snprintf(name
, 63, "%s%d", proc_dma_ring
, dma_ctx
->device_num
);
remove_proc_entry(name
, NULL
);