Commit | Line | Data |
---|---|---|
800f879a AT |
1 | /* |
2 | * Copyright 2010-2017 Intel Corporation. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License, version 2, | |
6 | * as published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
11 | * General Public License for more details. | |
12 | * | |
13 | * Disclaimer: The codes contained in these modules may be specific to | |
14 | * the Intel Software Development Platform codenamed Knights Ferry, | |
15 | * and the Intel product codenamed Knights Corner, and are not backward | |
16 | * compatible with other Intel products. Additionally, Intel will NOT | |
17 | * support the codes or instruction set in future products. | |
18 | * | |
19 | * Intel offers no warranty of any kind regarding the code. This code is | |
20 | * licensed on an "AS IS" basis and Intel is not obligated to provide | |
21 | * any support, assistance, installation, training, or other services | |
22 | * of any kind. Intel is also not obligated to provide any updates, | |
23 | * enhancements or extensions. Intel specifically disclaims any warranty | |
24 | * of merchantability, non-infringement, fitness for any particular | |
25 | * purpose, and any other warranty. | |
26 | * | |
27 | * Further, Intel disclaims all liability of any kind, including but | |
28 | * not limited to liability for infringement of any proprietary rights, | |
29 | * relating to the use of the code, even if Intel is notified of the | |
30 | * possibility of such liability. Except as expressly stated in an Intel | |
31 | * license agreement provided with this code and agreed upon with Intel, | |
32 | * no license, express or implied, by estoppel or otherwise, to any | |
33 | * intellectual property rights is granted herein. | |
34 | */ | |
35 | ||
36 | #include<linux/module.h> | |
37 | #include<linux/init.h> | |
38 | #include<linux/slab.h> | |
39 | #include<asm/io.h> | |
40 | #include<linux/mm.h> | |
41 | #include<linux/kernel.h> | |
42 | #include<linux/interrupt.h> | |
43 | #include<linux/proc_fs.h> | |
44 | #include<linux/bitops.h> | |
45 | #include<linux/version.h> | |
46 | #ifdef _MIC_SCIF_ | |
47 | #include <asm/mic/mic_common.h> | |
48 | #ifdef CONFIG_PAGE_CACHE_DMA | |
49 | #include <linux/mic_dma/mic_dma_callback.h> | |
50 | #endif | |
51 | #endif | |
52 | ||
53 | #ifndef _MIC_SCIF_ | |
54 | #include <mic/micscif.h> | |
55 | #include "mic_common.h" | |
56 | #endif | |
57 | ||
58 | #include <mic/mic_dma_lib.h> | |
59 | #include <mic/micscif_smpt.h> | |
60 | #include <mic/mic_dma_md.h> | |
61 | #include <mic/mic_dma_api.h> | |
62 | #include <mic/compl_buf_ring.h> | |
63 | #include <mic/micscif_smpt.h> | |
64 | #include <mic/micsboxdefine.h> | |
65 | ||
66 | MODULE_LICENSE("GPL"); | |
67 | ||
68 | #ifdef MIC_IS_EMULATION | |
69 | #define DMA_TO (INT_MAX) | |
70 | #define DMA_FENCE_TIMEOUT_CNT (INT_MAX) | |
71 | #else | |
72 | #define DMA_TO (5 * HZ) | |
73 | #define DMA_SLOWEST_BW (300) // 300Mbps | |
74 | // the maximum size for each decriptor entry is 2M | |
75 | #define DMA_FENCE_TIMEOUT_CNT (2 * MIC_MAX_NUM_DESC_PER_RING /DMA_SLOWEST_BW/ (DMA_TO/HZ)) | |
76 | #endif | |
77 | ||
78 | #ifdef _MIC_SCIF_ | |
79 | #define MAX_DMA_XFER_SIZE MIC_MAX_DMA_XFER_SIZE | |
80 | #else | |
81 | /* Use 512K as the maximum descriptor transfer size for Host */ | |
82 | #define MAX_DMA_XFER_SIZE (((1U) * 1024 * 1024) >> 1) | |
83 | #endif | |
84 | #ifndef KASSERT | |
85 | #define KASSERT(x, y, ...) \ | |
86 | do { \ | |
87 | if(!x) \ | |
88 | printk(y, ##__VA_ARGS__);\ | |
89 | BUG_ON(!x); \ | |
90 | } while(0) | |
91 | #endif | |
92 | /* | |
93 | * Arrary of per device DMA contexts. The card only uses index 0. The host uses one | |
94 | * context per card starting from 0. | |
95 | */ | |
96 | static struct mic_dma_ctx_t *mic_dma_context[MAX_BOARD_SUPPORTED + 1]; | |
97 | static struct mutex lock_dma_dev_init[MAX_BOARD_SUPPORTED + 1]; | |
98 | ||
99 | enum mic_desc_format_type { | |
100 | NOP, | |
101 | MEMCOPY, | |
102 | STATUS, | |
103 | GENERAL, | |
104 | KEYNONCECNT, | |
105 | KEY | |
106 | }; | |
107 | char proc_dma_reg[]="mic_dma_registers_"; | |
108 | char proc_dma_ring[]="mic_dma_ring_"; | |
109 | ||
110 | #define PR_PREFIX "DMA_LIB_MI:" | |
111 | #define DMA_DESC_RING_SIZE MIC_MAX_NUM_DESC_PER_RING | |
112 | #define MAX_POLLING_BUFFERS DMA_DESC_RING_SIZE | |
113 | ||
114 | #define DMA_PROC | |
115 | static void mic_dma_proc_init(struct mic_dma_ctx_t *dma_ctx); | |
116 | static void mic_dma_proc_uninit(struct mic_dma_ctx_t *dma_ctx); | |
117 | ||
118 | /* | |
119 | * TODO: This is size of s/w interrupt ring. | |
120 | * We need to figure out a value so that we don't run out of memory in | |
121 | * interrupt ring and at the same time don't waste memory | |
122 | */ | |
123 | #define NUM_COMP_BUFS (((PAGE_SIZE/sizeof(struct dma_completion_cb*)) - 10) * 10) | |
124 | ||
125 | struct intr_compl_buf_ring { | |
126 | struct dma_completion_cb **comp_cb_array; | |
127 | struct compl_buf_ring ring; | |
128 | int old_tail; | |
129 | }; | |
130 | ||
131 | struct mic_dma_ctx_t; /* Forward Declaration */ | |
132 | ||
133 | struct dma_channel { | |
134 | int ch_num;/*Duplicated in md_mic_dma_chan struct too*/ | |
135 | struct md_mic_dma_chan *chan; | |
136 | atomic_t flags; | |
137 | wait_queue_head_t intr_wq; | |
138 | wait_queue_head_t access_wq; | |
139 | union md_mic_dma_desc *desc_ring_bak; | |
140 | union md_mic_dma_desc *desc_ring; | |
141 | phys_addr_t desc_ring_phys; | |
142 | uint64_t next_write_index; /* next write index into desc ring */ | |
143 | struct intr_compl_buf_ring intr_ring; | |
144 | struct compl_buf_ring poll_ring; | |
145 | struct mic_dma_ctx_t *dma_ctx; /* Pointer to parent DMA context */ | |
146 | }; | |
147 | ||
148 | /* Per MIC device (per MIC board) DMA context */ | |
149 | struct mic_dma_ctx_t { | |
150 | struct dma_channel dma_channels[MAX_NUM_DMA_CHAN]; | |
151 | int last_allocated_dma_channel_num; | |
152 | struct mic_dma_device dma_dev; | |
153 | int device_num; | |
154 | atomic_t ref_count; /* Reference count */ | |
155 | atomic_t ch_num; | |
156 | }; | |
157 | ||
158 | /* DMA Library Init/Uninit Routines */ | |
159 | static int mic_dma_lib_init(uint8_t *mmio_va_base, struct mic_dma_ctx_t *dma_ctx); | |
160 | static void mic_dma_lib_uninit(struct mic_dma_ctx_t *dma_ctx); | |
161 | ||
162 | int get_chan_num(struct dma_channel *chan) | |
163 | { | |
164 | return chan->ch_num; | |
165 | } | |
166 | EXPORT_SYMBOL(get_chan_num); | |
167 | ||
168 | void initdmaglobalvar(void) | |
169 | { | |
170 | memset(mic_dma_context, 0, sizeof(struct mic_dma_ctx_t *) * (MAX_BOARD_SUPPORTED + 1)); | |
171 | } | |
172 | ||
173 | static void | |
174 | ack_dma_interrupt(struct dma_channel *ch) | |
175 | { | |
176 | md_mic_dma_chan_mask_intr(&ch->dma_ctx->dma_dev, ch->chan); | |
177 | md_mic_dma_chan_unmask_intr(&ch->dma_ctx->dma_dev, ch->chan); | |
178 | } | |
179 | ||
180 | /* Returns true if the next write index is "within" bounds */ | |
181 | static inline bool verify_next_write_index(struct dma_channel *ch) | |
182 | { | |
183 | bool ret = false; | |
184 | ||
185 | if (ch->next_write_index < DMA_DESC_RING_SIZE) | |
186 | ret = true; | |
187 | else | |
188 | printk(KERN_ERR "%s %d OOB ch_num 0x%x next_write_index 0x%llx\n", | |
189 | __func__, __LINE__, | |
190 | ch->ch_num, ch->next_write_index); | |
191 | return ret; | |
192 | } | |
193 | ||
194 | /* TODO: | |
195 | * See if we can use __get_free_pages or something similar | |
196 | * get_free_pages expects a power of 2 number of pages | |
197 | */ | |
198 | static void | |
199 | alloc_dma_desc_ring_mem(struct dma_channel *ch, struct mic_dma_ctx_t *dma_ctx) | |
200 | { | |
201 | #ifndef _MIC_SCIF_ | |
202 | struct pci_dev *pdev; | |
203 | #endif | |
204 | /* Is there any kernel allocator which provides the | |
205 | * option to give the alignment?? | |
206 | */ | |
207 | ch->desc_ring = kzalloc( | |
208 | (DMA_DESC_RING_SIZE * sizeof(*ch->desc_ring)) + PAGE_SIZE, GFP_KERNEL); | |
209 | ch->desc_ring_bak = ch->desc_ring; | |
210 | ch->desc_ring = (union md_mic_dma_desc *)ALIGN( | |
211 | (uint64_t)ch->desc_ring, PAGE_SIZE); | |
212 | #ifdef _MIC_SCIF_ | |
213 | ch->desc_ring_phys = virt_to_phys(ch->desc_ring); | |
214 | #else | |
215 | micscif_pci_dev(dma_ctx->device_num, &pdev); | |
216 | ch->desc_ring_phys = mic_map_single(dma_ctx->device_num - 1, pdev, (void *)ch->desc_ring, | |
217 | (DMA_DESC_RING_SIZE * sizeof(*ch->desc_ring)) + PAGE_SIZE); | |
218 | BUG_ON(pci_dma_mapping_error(pdev, ch->desc_ring_phys)); | |
219 | #endif | |
220 | } | |
221 | ||
222 | /* | |
223 | * Call completion cb functions: | |
224 | * Take care of case where we allocated temp buf | |
225 | */ | |
226 | static void | |
227 | mic_dma_lib_interrupt_handler(struct dma_channel *chan) | |
228 | { | |
229 | int i = 0; | |
230 | int ring_size = chan->intr_ring.ring.size; | |
231 | struct dma_completion_cb **temp = chan->intr_ring.comp_cb_array; | |
232 | struct dma_completion_cb *cb; | |
233 | int new_tail, old_tail; | |
234 | ||
235 | if (mic_hw_family(chan->dma_ctx->device_num) == FAMILY_KNC && | |
236 | mic_hw_stepping(chan->dma_ctx->device_num) >= KNC_B0_STEP) { | |
237 | unsigned long error = *((uint32_t*)chan->chan->dstat_wb_loc); | |
238 | if (unlikely(test_bit(31, &error))) | |
239 | printk(KERN_ERR "DMA h/w error - %s %d, dstatwb=%lx\n", | |
240 | __func__, __LINE__, error); | |
241 | } | |
242 | new_tail = read_tail(&chan->intr_ring.ring); | |
243 | old_tail = chan->intr_ring.old_tail; | |
244 | ||
245 | for (; i < ring_size && old_tail != new_tail; | |
246 | old_tail = incr_rb_index(old_tail, ring_size), i++) { | |
247 | cb = (struct dma_completion_cb *)xchg(&temp[old_tail], NULL); | |
248 | if (cb) { | |
249 | cb->dma_completion_func(cb->cb_cookie); | |
250 | } | |
251 | } | |
252 | chan->intr_ring.old_tail = new_tail; | |
253 | update_tail(&chan->intr_ring.ring, new_tail); | |
254 | wake_up(&chan->intr_wq); | |
255 | if (i == ring_size && old_tail != new_tail) { | |
256 | printk(KERN_ERR PR_PREFIX "Something went wrong, old tail = %d, new tail = %d\n", | |
257 | old_tail, new_tail); | |
258 | } | |
259 | } | |
260 | ||
261 | #ifdef _MIC_SCIF_ | |
262 | /* | |
263 | * TODO; | |
264 | * Maybe move the logic into slow interrupt handler | |
265 | */ | |
266 | static irqreturn_t | |
267 | dma_interrupt_handler(int irq, void *dev_id) | |
268 | { | |
269 | struct dma_channel *chan = ((struct dma_channel*)dev_id); | |
270 | ||
271 | ack_dma_interrupt(chan); | |
272 | mic_dma_lib_interrupt_handler(chan); | |
273 | ||
274 | return IRQ_HANDLED; | |
275 | } | |
276 | #else | |
277 | ||
278 | #define SBOX_SICR0_DMA(x) (((x) >> 8) & 0xff) | |
279 | ||
280 | /* | |
281 | * TODO; | |
282 | * Maybe move the logic into slow interrupt handler | |
283 | */ | |
284 | void | |
285 | host_dma_interrupt_handler(mic_dma_handle_t dma_handle, uint32_t sboxSicr0reg) | |
286 | { | |
287 | struct mic_dma_ctx_t *dma_ctx = (struct mic_dma_ctx_t *) dma_handle; | |
288 | uint32_t dma_chan_id; | |
289 | struct dma_channel *ch; | |
290 | ||
291 | for (dma_chan_id = 0; dma_chan_id < 8; dma_chan_id++) { | |
292 | if (SBOX_SICR0_DMA(sboxSicr0reg) & (0x1 << dma_chan_id)) { | |
293 | ch = &dma_ctx->dma_channels[dma_chan_id]; | |
294 | if (ch->desc_ring) | |
295 | host_dma_lib_interrupt_handler(ch); | |
296 | } | |
297 | } | |
298 | } | |
299 | ||
300 | void | |
301 | host_dma_lib_interrupt_handler(struct dma_channel *chan) | |
302 | { | |
303 | ack_dma_interrupt(chan); | |
304 | mic_dma_lib_interrupt_handler(chan); | |
305 | } | |
306 | #endif | |
307 | ||
308 | static void | |
309 | mi_mic_dma_chan_setup(struct dma_channel *ch, struct mic_dma_ctx_t *dma_ctx) | |
310 | { | |
311 | ch->next_write_index = ch->chan->cached_tail; | |
312 | ||
313 | init_ring(&ch->poll_ring, MAX_POLLING_BUFFERS, dma_ctx->device_num); | |
314 | ||
315 | ch->intr_ring.comp_cb_array = | |
316 | kzalloc(sizeof(*ch->intr_ring.comp_cb_array) * NUM_COMP_BUFS, GFP_KERNEL); | |
317 | init_ring(&ch->intr_ring.ring, NUM_COMP_BUFS, dma_ctx->device_num); | |
318 | ch->intr_ring.old_tail = 0; | |
319 | } | |
320 | ||
321 | static void | |
322 | mi_mic_dma_chan_destroy(struct dma_channel *ch, struct mic_dma_ctx_t *dma_ctx) | |
323 | { | |
324 | uninit_ring(&ch->intr_ring.ring, dma_ctx->device_num); | |
325 | kfree(ch->intr_ring.comp_cb_array); | |
326 | uninit_ring(&ch->poll_ring, dma_ctx->device_num); | |
327 | } | |
328 | ||
329 | int | |
330 | open_dma_device(int device_num, uint8_t *mmio_va_base, mic_dma_handle_t* dma_handle) | |
331 | { | |
332 | int result = 0; | |
333 | ||
334 | if (device_num >= MAX_BOARD_SUPPORTED) | |
335 | return -EINVAL; | |
336 | ||
337 | mutex_lock(&lock_dma_dev_init[device_num]); | |
338 | if (!mic_dma_context[device_num]) { | |
339 | mic_dma_context[device_num] = kzalloc(sizeof(struct mic_dma_ctx_t), GFP_KERNEL); | |
340 | BUG_ON(!mic_dma_context[device_num]); | |
341 | ||
342 | mic_dma_context[device_num]->device_num = device_num; | |
343 | ||
344 | result = mic_dma_lib_init(mmio_va_base, mic_dma_context[device_num]); | |
345 | BUG_ON(result); | |
346 | } | |
347 | ||
348 | atomic_inc(&mic_dma_context[device_num]->ref_count); | |
349 | *dma_handle = mic_dma_context[device_num]; | |
350 | mutex_unlock(&lock_dma_dev_init[device_num]); | |
351 | ||
352 | return result; | |
353 | } | |
354 | EXPORT_SYMBOL(open_dma_device); | |
355 | ||
356 | void | |
357 | close_dma_device(int device_num, mic_dma_handle_t *dma_handle) | |
358 | { | |
359 | struct mic_dma_ctx_t *dma_ctx; | |
360 | ||
361 | if (device_num >= MAX_BOARD_SUPPORTED) | |
362 | return; | |
363 | ||
364 | mutex_lock(&lock_dma_dev_init[device_num]); | |
365 | dma_ctx = (struct mic_dma_ctx_t *) *dma_handle; | |
366 | if (dma_ctx && | |
367 | atomic_read(&dma_ctx->ref_count) && | |
368 | atomic_dec_and_test(&dma_ctx->ref_count)) { | |
369 | mic_dma_lib_uninit(dma_ctx); | |
370 | mic_dma_context[dma_ctx->device_num] = 0; | |
371 | *dma_handle = NULL; | |
372 | kfree(dma_ctx); | |
373 | } | |
374 | mutex_unlock(&lock_dma_dev_init[device_num]); | |
375 | } | |
376 | EXPORT_SYMBOL(close_dma_device); | |
377 | ||
378 | void mi_mic_dma_chan_set_dstat_wb(struct mic_dma_ctx_t *dma_ctx, | |
379 | struct md_mic_dma_chan *chan) | |
380 | { | |
381 | #ifndef _MIC_SCIF_ | |
382 | struct pci_dev *pdev; | |
383 | #endif | |
384 | if (!chan->dstat_wb_phys) { | |
385 | chan->dstat_wb_loc = kzalloc(sizeof(uint32_t), GFP_KERNEL); | |
386 | ||
387 | #ifdef _MIC_SCIF_ | |
388 | chan->dstat_wb_phys = virt_to_phys(chan->dstat_wb_loc); | |
389 | #else | |
390 | micscif_pci_dev(dma_ctx->device_num, &pdev); | |
391 | chan->dstat_wb_phys = mic_map_single(dma_ctx->device_num - 1, pdev, chan->dstat_wb_loc, | |
392 | sizeof(uint32_t)); | |
393 | BUG_ON(pci_dma_mapping_error(pdev, chan->dstat_wb_phys)); | |
394 | #endif | |
395 | } | |
396 | md_mic_dma_chan_set_dstat_wb(&dma_ctx->dma_dev, chan); | |
397 | } | |
398 | ||
399 | void | |
400 | md_mic_dma_chan_setup(struct mic_dma_ctx_t *dma_ctx, struct dma_channel *ch) | |
401 | { | |
402 | md_mic_dma_chan_unmask_intr(&dma_ctx->dma_dev, ch->chan); | |
403 | ||
404 | /* | |
405 | * Disable the channel, update desc ring base and size, write new head | |
406 | * and then enable the channel. | |
407 | */ | |
408 | if (mic_hw_family(ch->dma_ctx->device_num) == FAMILY_KNC && | |
409 | mic_hw_stepping(ch->dma_ctx->device_num) >= KNC_B0_STEP) { | |
410 | mi_mic_dma_chan_set_dstat_wb(dma_ctx, ch->chan); | |
411 | md_mic_dma_chan_set_dcherr_msk(&dma_ctx->dma_dev, ch->chan, 0); | |
412 | } | |
413 | md_mic_dma_chan_set_desc_ring(&dma_ctx->dma_dev, ch->chan, | |
414 | ch->desc_ring_phys, | |
415 | DMA_DESC_RING_SIZE); | |
416 | ||
417 | wmb(); | |
418 | ||
419 | md_mic_dma_chan_unmask_intr(&dma_ctx->dma_dev, ch->chan); | |
420 | } | |
421 | ||
422 | int | |
423 | mic_dma_lib_init(uint8_t *mmio_va_base, struct mic_dma_ctx_t *dma_ctx) | |
424 | { | |
425 | int i; | |
426 | #ifdef _MIC_SCIF_ | |
427 | int ret_value; | |
428 | #endif | |
429 | struct dma_channel *ch; | |
430 | enum md_mic_dma_chan_owner owner, currentOwner; | |
431 | ||
432 | //pr_debug(PR_PREFIX "Initialized the dma mmio va=%p\n", mmio_va_base); | |
433 | // Using this to check where the DMA lib is at for now. | |
434 | currentOwner = mmio_va_base == 0 ? MIC_DMA_CHAN_MIC_OWNED : MIC_DMA_CHAN_HOST_OWNED; | |
435 | ||
436 | // TODO: multi-card support | |
437 | md_mic_dma_init(&dma_ctx->dma_dev, mmio_va_base); | |
438 | ||
439 | for (i = 0 ; i < MAX_NUM_DMA_CHAN; i++) { | |
440 | ch = &dma_ctx->dma_channels[i]; | |
441 | ||
442 | /* Initialize pointer to parent */ | |
443 | ch->dma_ctx = dma_ctx; | |
444 | ||
445 | owner = i > __LAST_HOST_CHAN_NUM ? MIC_DMA_CHAN_MIC_OWNED | |
446 | : MIC_DMA_CHAN_HOST_OWNED; | |
447 | ||
448 | // This has to be done from card side | |
449 | ch->chan = md_mic_dma_request_chan(&dma_ctx->dma_dev, owner); | |
450 | KASSERT((ch->chan != NULL), "dummy\n"); | |
451 | ch->ch_num = ch->chan->ch_num; | |
452 | ||
453 | #ifdef _MIC_SCIF_ | |
454 | /* | |
455 | * Host driver would have executed by now and thus setup the | |
456 | * desc. ring | |
457 | */ | |
458 | if (ch->chan->owner == MIC_DMA_CHAN_HOST_OWNED) | |
459 | md_mic_dma_enable_chan(&dma_ctx->dma_dev, i, true); | |
460 | #endif | |
461 | ||
462 | atomic_set(&(ch->flags), CHAN_INUSE); // Mark as used by default | |
463 | if (currentOwner == owner) { | |
464 | alloc_dma_desc_ring_mem(ch, dma_ctx); | |
465 | ||
466 | #ifdef _MIC_SCIF_ // DMA now shares the IRQ handler with other system interrupts | |
467 | ret_value = request_irq(i, dma_interrupt_handler, IRQF_DISABLED, | |
468 | "dma channel", ch); | |
469 | ret_value = ret_value; | |
470 | //pr_debug(PR_PREFIX "Interrupt handler ret value for chan %d = %d\n", i, ret_value); | |
471 | #endif | |
472 | md_mic_dma_chan_setup(dma_ctx, ch); | |
473 | ||
474 | mi_mic_dma_chan_setup(ch, dma_ctx); | |
475 | ||
476 | init_waitqueue_head(&ch->intr_wq); | |
477 | init_waitqueue_head(&ch->access_wq); | |
478 | // Only mark owned channel to be available | |
479 | atomic_set(&(ch->flags), CHAN_AVAILABLE); | |
480 | md_mic_dma_print_debug(&dma_ctx->dma_dev, ch->chan); | |
481 | } else { | |
482 | ch->desc_ring = NULL; | |
483 | } | |
484 | } | |
485 | ||
486 | /* Initialize last_allocated_dma_channel */ | |
487 | dma_ctx->last_allocated_dma_channel_num = -1; | |
488 | //pr_debug(PR_PREFIX "Initialized the dma channels\n"); | |
489 | mic_dma_proc_init(dma_ctx); | |
490 | return 0; | |
491 | } | |
492 | ||
493 | void | |
494 | mic_dma_lib_uninit(struct mic_dma_ctx_t *dma_ctx) | |
495 | { | |
496 | int i; | |
497 | struct dma_channel *ch; | |
498 | #ifndef _MIC_SCIF_ | |
499 | struct pci_dev *pdev; | |
500 | #endif | |
501 | ||
502 | mic_dma_proc_uninit(dma_ctx); | |
503 | for (i = 0 ; i < MAX_NUM_DMA_CHAN; i++) { | |
504 | ch = &dma_ctx->dma_channels[i]; | |
505 | if (!ch->desc_ring) | |
506 | continue; | |
507 | drain_dma_intr(ch); | |
508 | /* Request the channel but don't free it. Errors are okay */ | |
509 | request_dma_channel(ch); | |
510 | #ifdef _MIC_SCIF_ // DMA now shares the IRQ handler with other system interrupts | |
511 | free_irq(i, ch); | |
512 | #endif | |
513 | mi_mic_dma_chan_destroy(ch, dma_ctx); | |
514 | #ifndef _MIC_SCIF_ | |
515 | micscif_pci_dev(dma_ctx->device_num, &pdev); | |
516 | mic_unmap_single(dma_ctx->device_num - 1, pdev, ch->desc_ring_phys, | |
517 | (DMA_DESC_RING_SIZE * sizeof(*ch->desc_ring)) + PAGE_SIZE); | |
518 | #endif | |
519 | ||
520 | kfree(ch->desc_ring_bak); | |
521 | ch->desc_ring_bak = NULL; | |
522 | ch->desc_ring = NULL; | |
523 | if (mic_hw_family(ch->dma_ctx->device_num) == FAMILY_KNC && | |
524 | mic_hw_stepping(dma_ctx->device_num) >= KNC_B0_STEP) { | |
525 | #ifndef _MIC_SCIF_ | |
526 | mic_unmap_single(dma_ctx->device_num - 1, pdev, ch->chan->dstat_wb_phys, | |
527 | sizeof(uint32_t)); | |
528 | #endif | |
529 | kfree(ch->chan->dstat_wb_loc); | |
530 | ch->chan->dstat_wb_loc = NULL; | |
531 | ch->chan->dstat_wb_phys = 0; | |
532 | } | |
533 | md_mic_dma_free_chan(&dma_ctx->dma_dev, ch->chan); | |
534 | } | |
535 | #ifndef MIC_IS_EMULATION | |
536 | /* Ensure that all waiters for DMA channels time out */ | |
537 | msleep(DMA_TO/HZ * 1000); | |
538 | #endif | |
539 | md_mic_dma_uninit(&dma_ctx->dma_dev); | |
540 | //pr_debug(PR_PREFIX "Uninitialized the dma channels\n"); | |
541 | } | |
542 | ||
543 | /* | |
544 | * reserve_dma_channel - reserve a given dma channel for exclusive use | |
545 | * | |
546 | * @dma_handle - handle to DMA device returned by open_dma_device | |
547 | * @chan_num - Channel number to be reserved | |
548 | * @chan - set to point to the dma channel reserved by the call | |
549 | * | |
550 | * Returns < 1 on error (errorno) | |
551 | * Returns 0 on success | |
552 | * | |
553 | * NOTES: Should this function sleep waiting for the lock? | |
554 | * TODO: | |
555 | * Maybe there should be a blocking and non-blocking versions of this function | |
556 | */ | |
557 | int | |
558 | reserve_dma_channel(mic_dma_handle_t dma_handle, int chan_num, struct dma_channel **chan) | |
559 | { | |
560 | struct mic_dma_ctx_t *dma_ctx = (struct mic_dma_ctx_t *) dma_handle; | |
561 | ||
562 | /* | |
563 | * Do we need to do acquire the lock for statically allocated channels? | |
564 | * I am assuming we dont have to lock | |
565 | */ | |
566 | if (CHAN_AVAILABLE == atomic_cmpxchg(&(dma_ctx->dma_channels[chan_num].flags), | |
567 | CHAN_AVAILABLE, CHAN_INUSE)) { | |
568 | *chan = &dma_ctx->dma_channels[chan_num]; | |
569 | return 0; | |
570 | } | |
571 | return -1; | |
572 | } | |
573 | EXPORT_SYMBOL(reserve_dma_channel); | |
574 | ||
575 | /* | |
576 | * allocate_dma_channel - dynamically allocate a dma channel (for a short while). Will | |
577 | * search for, choose, and lock down one channel for use by the calling thread. | |
578 | * | |
579 | * @dma_handle - handle to DMA device returned by open_dma_device | |
580 | * @chan - Returns the dma_channel pointer that was allocated by the call | |
581 | * | |
582 | * Returns < 1 on error | |
583 | * Returns 0 on success | |
584 | * | |
585 | * NOTE: This function grabs a lock before exiting -- the calling thread MUST NOT | |
586 | * sleep, and must call free_dma_channel before returning to user-space or switching | |
587 | * volantarily to another thread. Similarly, this function cannot be called from | |
588 | * an interrupt context at this time. | |
589 | * | |
590 | * TODO: How do we pick a dma channel? | |
591 | * For now I am doing it in round robin fashion. | |
592 | */ | |
593 | int | |
594 | allocate_dma_channel(mic_dma_handle_t dma_handle, struct dma_channel **chan) | |
595 | { | |
596 | struct mic_dma_ctx_t *dma_ctx = (struct mic_dma_ctx_t *) dma_handle; | |
597 | int i, j; | |
598 | ||
599 | if (!dma_ctx) | |
600 | return -ENODEV; | |
601 | ||
602 | j = dma_ctx->last_allocated_dma_channel_num + 1; | |
603 | ||
604 | for (i = 0; i < MAX_NUM_DMA_CHAN; i++, j++) { | |
605 | if (CHAN_AVAILABLE == atomic_cmpxchg(&(dma_ctx->dma_channels[j % | |
606 | MAX_NUM_DMA_CHAN].flags), | |
607 | CHAN_AVAILABLE, CHAN_INUSE)) { | |
608 | *chan = &(dma_ctx->dma_channels[j % MAX_NUM_DMA_CHAN]); | |
609 | dma_ctx->last_allocated_dma_channel_num = j % MAX_NUM_DMA_CHAN; | |
610 | return 0; | |
611 | } | |
612 | } | |
613 | return -1; | |
614 | } | |
615 | EXPORT_SYMBOL(allocate_dma_channel); | |
616 | ||
617 | /* | |
618 | * request_dma_channel - Request a specific DMA channel. | |
619 | * | |
620 | * @chan - Returns the dma_channel pointer that was requested | |
621 | * | |
622 | * Returns: 0 on success and -ERESTARTSYS if the wait was interrupted | |
623 | * or -EBUSY if the channel was not available. | |
624 | * | |
625 | * NOTE: This function must call free_dma_channel before returning to | |
626 | * user-space. | |
627 | */ | |
628 | int request_dma_channel(struct dma_channel *chan) | |
629 | { | |
630 | int ret; | |
631 | ||
632 | ret = wait_event_interruptible_timeout(chan->access_wq, | |
633 | CHAN_AVAILABLE == atomic_cmpxchg(&chan->flags, | |
634 | CHAN_AVAILABLE, CHAN_INUSE), DMA_TO); | |
635 | if (!ret) { | |
636 | printk(KERN_ERR "%s %d TO chan 0x%x\n", __func__, __LINE__, chan->ch_num); | |
637 | ret = -EBUSY; | |
638 | } | |
639 | if (ret > 0) | |
640 | ret = 0; | |
641 | return ret; | |
642 | } | |
643 | EXPORT_SYMBOL(request_dma_channel); | |
644 | ||
645 | /* | |
646 | * free_dma_channel - after allocating a channel, used to | |
647 | * free the channel after DMAs are submitted | |
648 | * | |
649 | * @chan - pointer to the dma_channel struct that was allocated | |
650 | * | |
651 | * Returns 0 on success, < 1 on error (errorno) | |
652 | * | |
653 | * NOTE: This function must be called after all do_dma calls are finished, | |
654 | * but can be called before the DMAs actually complete (as long as the comp_cb() | |
655 | * handler in do_dma don't refer to the dma_channel struct). If called with a | |
656 | * dynamically allocated dma_chan, the caller must be the thread that called | |
657 | * allocate_dma_chan. When operating on a dynamic channel, free unlocks the | |
658 | * mutex locked in allocate. Statically allocated channels cannot be freed, | |
659 | * and calling this function with that type of channel will return an error. | |
660 | */ | |
661 | int | |
662 | free_dma_channel(struct dma_channel *chan) | |
663 | { | |
664 | /* | |
665 | * Why can't we use this function with channels that were statically allocated?? | |
666 | */ | |
667 | BUG_ON(CHAN_INUSE != | |
668 | atomic_cmpxchg(&chan->flags, CHAN_INUSE, CHAN_AVAILABLE)); | |
669 | wake_up(&chan->access_wq); | |
670 | return 0; | |
671 | } | |
672 | EXPORT_SYMBOL(free_dma_channel); | |
673 | ||
674 | static __always_inline uint32_t | |
675 | get_dma_tail_pointer(struct dma_channel *chan) | |
676 | { | |
677 | struct mic_dma_device *dma_dev; | |
678 | dma_dev = &chan->dma_ctx->dma_dev; | |
679 | return md_mic_dma_chan_read_tail(dma_dev, chan->chan); | |
680 | } | |
681 | /* | |
682 | * Return -1 in case of error | |
683 | */ | |
684 | static int | |
685 | program_memcpy_descriptors(struct dma_channel *chan, uint64_t src, uint64_t dst, size_t len) | |
686 | { | |
687 | size_t current_transfer_len; | |
688 | bool is_astep = false; | |
689 | unsigned long ts = jiffies; | |
690 | ||
691 | if (mic_hw_family(chan->dma_ctx->device_num) == FAMILY_KNC) { | |
692 | if (mic_hw_stepping(chan->dma_ctx->device_num) == KNC_A_STEP) | |
693 | is_astep = true; | |
694 | } else { | |
695 | is_astep = true; | |
696 | } | |
697 | do { | |
698 | current_transfer_len = (len > MAX_DMA_XFER_SIZE) ? | |
699 | MAX_DMA_XFER_SIZE : len; | |
700 | ||
701 | ts = jiffies; | |
702 | while (!md_avail_desc_ring_space(&chan->dma_ctx->dma_dev, is_astep, chan->chan, | |
703 | (uint32_t)chan->next_write_index, 1)) { | |
704 | if (time_after(jiffies,ts + DMA_TO)) { | |
705 | printk(KERN_ERR "%s %d TO chan 0x%x\n", __func__, __LINE__, chan->ch_num); | |
706 | return -ENOMEM; | |
707 | } | |
708 | } | |
709 | ||
710 | //pr_debug("src_phys=0x%llx, dst_phys=0x%llx, size=0x%zx\n", src_phys_addr, dst_phys_addr, current_transfer_len); | |
711 | md_mic_dma_memcpy_desc(&chan->desc_ring[chan->next_write_index], | |
712 | src, dst, current_transfer_len); | |
713 | chan->next_write_index = incr_rb_index((int)chan->next_write_index, | |
714 | chan->chan->num_desc_in_ring); | |
715 | len -= current_transfer_len; | |
716 | dst = dst + current_transfer_len; | |
717 | src = src + current_transfer_len; | |
718 | } while(len > 0); | |
719 | ||
720 | return 0; | |
721 | } | |
722 | ||
723 | /* | |
724 | * do_dma - main dma function: perform a dma memcpy, len bytes from src to dst | |
725 | * | |
726 | * @chan - DMA channel to use for the transfer. The channel can be allocated | |
727 | * dynamically by calling allocate_dma_chan, or statically by | |
728 | * reserve_dma_chan. Using a channel not allocated in this way will | |
729 | * result in undefined behavior. | |
730 | * @flags - ATOMIC, called from an interrupt context (no blocking) | |
731 | * @src - src physical address | |
732 | * @dst - dst physical address | |
733 | * @len - Length of the dma | |
734 | * @comp_cb - When the DMA is complete, the struct's function will be called. NOTE! | |
735 | * comp_cb(cb_cookie) is called from an interrupt context, so the | |
736 | * function must not sleep or block. | |
737 | * | |
738 | * TODO: Figure out proper value instead of -2 | |
739 | * Return < 0 on error | |
740 | * Return = -2 copy was done successfully, no need to wait | |
741 | * Return >= 0: DMA has been queued. Return value can be polled on for completion | |
742 | * if DO_DMA_POLLING was sent in flags | |
743 | * (poll cookie). An example (simplified w/ no error handling). | |
744 | * int cookie = do_dma(...); | |
745 | * while (poll_dma_completion(cookie) == 0); | |
746 | * printf("DMA now complete\n"); | |
747 | */ | |
748 | int | |
749 | do_dma(struct dma_channel *chan, int flags, uint64_t src, | |
750 | uint64_t dst, size_t len, struct dma_completion_cb *comp_cb) | |
751 | { | |
752 | /* | |
753 | * TODO: | |
754 | * Do we need to assert the ownership of channel?? | |
755 | */ | |
756 | int poll_ring_index = -1; | |
757 | int intr_ring_index = -1; | |
758 | uint32_t num_status_desc = 0; | |
759 | bool is_astep = false; | |
760 | unsigned long ts = jiffies; | |
761 | ||
762 | might_sleep(); | |
763 | if (flags & DO_DMA_INTR && !comp_cb) | |
764 | return -EINVAL; | |
765 | ||
766 | if (!verify_next_write_index(chan)) | |
767 | return -ENODEV; | |
768 | ||
769 | //pr_debug(PR_PREFIX "Current transfer src = 0x%llx,dst = 0x%llx, len = 0x%zx\n", src, dst, len); | |
770 | if (flags & DO_DMA_INTR) { | |
771 | int err; | |
772 | err = wait_event_interruptible_timeout(chan->intr_wq, | |
773 | (-1 != (intr_ring_index = allocate_buffer(&chan->intr_ring.ring))), | |
774 | DMA_TO); | |
775 | if (!err) { | |
776 | printk(KERN_ERR "%s %d TO chan 0x%x\n", __func__, __LINE__, chan->ch_num); | |
777 | err = -ENOMEM; | |
778 | } | |
779 | if (err > 0) | |
780 | err = 0; | |
781 | if (!err) { | |
782 | chan->intr_ring.comp_cb_array[intr_ring_index] = comp_cb; | |
783 | num_status_desc++; | |
784 | #ifdef CONFIG_MK1OM | |
785 | num_status_desc++; | |
786 | #endif | |
787 | } else { | |
788 | return err; | |
789 | } | |
790 | //pr_debug(PR_PREFIX "INTR intr_ring_index=%d, chan_num=%lx\n", intr_ring_index, (chan - dma_channels)); | |
791 | } | |
792 | ||
793 | if (flags & DO_DMA_POLLING) { | |
794 | poll_ring_index = allocate_buffer(&chan->poll_ring); | |
795 | if (-1 == poll_ring_index) | |
796 | return -ENOMEM; | |
797 | num_status_desc++; | |
798 | //pr_debug(PR_PREFIX "polling poll_ring_index=%d\n", poll_ring_index); | |
799 | } | |
800 | if (len && -ENOMEM == program_memcpy_descriptors(chan, src, dst, len)) { | |
801 | //pr_debug(PR_PREFIX "ERROR: do_dma: No available space from program_memcpy_descriptors\n"); | |
802 | return -ENOMEM; | |
803 | } | |
804 | ||
805 | if (mic_hw_family(chan->dma_ctx->device_num) == FAMILY_KNC) { | |
806 | if (mic_hw_stepping(chan->dma_ctx->device_num) == KNC_A_STEP) | |
807 | is_astep = true; | |
808 | } else { | |
809 | is_astep = true; | |
810 | } | |
811 | ||
812 | ts = jiffies; | |
813 | ||
814 | while (num_status_desc && num_status_desc > md_avail_desc_ring_space(&chan->dma_ctx->dma_dev, | |
815 | is_astep, chan->chan, (uint32_t)chan->next_write_index, num_status_desc)) { | |
816 | if (time_after(jiffies,ts + DMA_TO)) { | |
817 | printk(KERN_ERR "%s %d TO chan 0x%x\n", __func__, __LINE__, chan->ch_num); | |
818 | return -ENOMEM; | |
819 | } | |
820 | //pr_debug(PR_PREFIX "ERROR: do_dma: No available space from md_avail_desc_ring_space\n"); | |
821 | } | |
822 | ||
823 | if (flags & DO_DMA_POLLING) { | |
824 | incr_head(&chan->poll_ring); | |
825 | md_mic_dma_prep_status_desc(&chan->desc_ring[chan->next_write_index], | |
826 | poll_ring_index, | |
827 | chan->poll_ring.tail_phys, | |
828 | false); | |
829 | chan->next_write_index = incr_rb_index((int)chan->next_write_index, | |
830 | chan->chan->num_desc_in_ring); | |
831 | } | |
832 | ||
833 | if (flags & DO_DMA_INTR) { | |
834 | incr_head(&chan->intr_ring.ring); | |
835 | #ifdef CONFIG_MK1OM | |
836 | md_mic_dma_prep_status_desc(&chan->desc_ring[chan->next_write_index], | |
837 | intr_ring_index, | |
838 | chan->intr_ring.ring.tail_phys, | |
839 | false); | |
840 | chan->next_write_index = incr_rb_index((int)chan->next_write_index, | |
841 | chan->chan->num_desc_in_ring); | |
842 | #endif | |
843 | md_mic_dma_prep_status_desc(&chan->desc_ring[chan->next_write_index], | |
844 | intr_ring_index, | |
845 | chan->intr_ring.ring.tail_phys, | |
846 | true); | |
847 | chan->next_write_index = incr_rb_index((int)chan->next_write_index, | |
848 | chan->chan->num_desc_in_ring); | |
849 | } | |
850 | ||
851 | /* | |
852 | * TODO: | |
853 | * Maybe it is better if we update the head pointer for every descriptor?? | |
854 | */ | |
855 | md_mic_dma_chan_write_head(&chan->dma_ctx->dma_dev, chan->chan, (uint32_t)chan->next_write_index); | |
856 | //pr_debug(PR_PREFIX "in HW chan->next_write_index=%lld\n", chan->next_write_index); | |
857 | ||
858 | if (DO_DMA_POLLING & flags) | |
859 | return poll_ring_index; | |
860 | return 0; | |
861 | } | |
862 | EXPORT_SYMBOL(do_dma); | |
863 | ||
864 | /* | |
865 | * poll_dma_completion - check if a DMA is complete | |
866 | * | |
867 | * @poll_cookie - value returned from do_dma | |
868 | * | |
869 | * Returns | |
870 | * 0 -> DMA pending | |
871 | * 1 -> DMA completed | |
872 | * | |
873 | * Note: This is mostly useful after calling do_dma with a NULL comp_cb parameter, as | |
874 | * it will allow the caller to wait for DMA completion. | |
875 | */ | |
876 | int | |
877 | poll_dma_completion(int poll_cookie, struct dma_channel *chan) | |
878 | { | |
879 | if (!chan) | |
880 | return -EINVAL; | |
881 | /* | |
882 | * In case of interrupts the ISR runs and reads the value | |
883 | * of the tail location. If we are polling then we need | |
884 | * to read the value of the tail location before checking | |
885 | * if the entry is processed. | |
886 | */ | |
887 | chan->poll_ring.tail = read_tail(&chan->poll_ring); | |
888 | return is_entry_processed(&chan->poll_ring, poll_cookie); | |
889 | } | |
890 | EXPORT_SYMBOL(poll_dma_completion); | |
891 | ||
892 | /* | |
893 | * do_status_update: Update physical address location with the value provided. | |
894 | * Ensures all previous DMA descriptors submitted on this DMA | |
895 | * channel are executed. | |
896 | * @chan - DMA channel to use for the transfer. The channel can be allocated | |
897 | * dynamically by calling allocate_dma_channel, or statically by | |
898 | * reserve_dma_channel. Using a channel not allocated in this way will | |
899 | * result in undefined behavior. | |
900 | * @phys - physical address | |
901 | * @value - Value to be programmed | |
902 | */ | |
903 | int do_status_update(struct dma_channel *chan, uint64_t phys, uint64_t value) | |
904 | { | |
905 | unsigned long ts = jiffies; | |
906 | bool is_astep = false; | |
907 | ||
908 | if (!verify_next_write_index(chan)) | |
909 | return -ENODEV; | |
910 | ||
911 | if (mic_hw_family(chan->dma_ctx->device_num) == FAMILY_KNC) { | |
912 | if (mic_hw_stepping(chan->dma_ctx->device_num) == KNC_A_STEP) | |
913 | is_astep = true; | |
914 | } else { | |
915 | is_astep = true; | |
916 | } | |
917 | /* | |
918 | * TODO: | |
919 | * Do we need to assert the ownership of channel?? | |
920 | */ | |
921 | ts = jiffies; | |
922 | while (!md_avail_desc_ring_space(&chan->dma_ctx->dma_dev, | |
923 | is_astep, chan->chan, (uint32_t) chan->next_write_index, 1)) { | |
924 | cpu_relax(); | |
925 | if (time_after(jiffies,ts + DMA_TO)) { | |
926 | printk(KERN_ERR "%s %d TO chan 0x%x\n", __func__, __LINE__, chan->ch_num); | |
927 | return -EBUSY; | |
928 | } | |
929 | } | |
930 | ||
931 | md_mic_dma_prep_status_desc(&chan->desc_ring[chan->next_write_index], | |
932 | value, | |
933 | phys, | |
934 | false); | |
935 | ||
936 | chan->next_write_index = incr_rb_index((int)chan->next_write_index, | |
937 | chan->chan->num_desc_in_ring); | |
938 | ||
939 | md_mic_dma_chan_write_head(&chan->dma_ctx->dma_dev, | |
940 | chan->chan, (uint32_t)chan->next_write_index); | |
941 | return 0; | |
942 | } | |
943 | EXPORT_SYMBOL(do_status_update); | |
944 | ||
945 | /* | |
946 | * get_dma_mark: Obtain current value of DMA mark | |
947 | * @chan - DMA channel to use for the transfer. The channel can be allocated | |
948 | * dynamically by calling allocate_dma_channel, or statically by | |
949 | * reserve_dma_channel. Using a channel not allocated in this way will | |
950 | * result in undefined behavior. | |
951 | */ | |
952 | int get_dma_mark(struct dma_channel *chan) | |
953 | { | |
954 | if (chan) | |
955 | return chan->intr_ring.ring.head; | |
956 | else | |
957 | return -1; | |
958 | } | |
959 | EXPORT_SYMBOL(get_dma_mark); | |
960 | ||
961 | /* | |
962 | * program_dma_mark: Increment the current value of the DMA mark for a DMA channel | |
963 | * and program an interrupt status update descriptor which ensures that all DMA | |
964 | * descriptors programmed uptil this point in time are completed. | |
965 | * @chan - DMA channel to use for the transfer. The channel can be allocated | |
966 | * dynamically by calling allocate_dma_channel, or statically by | |
967 | * reserve_dma_channel. Using a channel not allocated in this way will | |
968 | * result in undefined behavior. | |
969 | */ | |
970 | int program_dma_mark(struct dma_channel *chan) | |
971 | { | |
972 | /* | |
973 | * TODO: | |
974 | * Do we need to assert the ownership of channel?? | |
975 | */ | |
976 | int intr_ring_index; | |
977 | int err; | |
978 | unsigned long ts = jiffies; | |
979 | uint32_t num_status_desc = 1; | |
980 | bool is_astep = false; | |
981 | ||
982 | if (!verify_next_write_index(chan)) | |
983 | return -ENODEV; | |
984 | ||
985 | if (mic_hw_family(chan->dma_ctx->device_num) == FAMILY_KNC) { | |
986 | if (mic_hw_stepping(chan->dma_ctx->device_num) == KNC_A_STEP) | |
987 | is_astep = true; | |
988 | } else { | |
989 | is_astep = true; | |
990 | } | |
991 | might_sleep(); | |
992 | err = wait_event_interruptible_timeout(chan->intr_wq, | |
993 | (-1 != (intr_ring_index = allocate_buffer(&chan->intr_ring.ring))), | |
994 | DMA_TO); | |
995 | if (!err) | |
996 | err = -EBUSY; | |
997 | if (err > 0) | |
998 | err = 0; | |
999 | if (err) | |
1000 | return err; | |
1001 | ||
1002 | #ifdef CONFIG_MK1OM | |
1003 | num_status_desc++; | |
1004 | #endif | |
1005 | ts = jiffies; | |
1006 | while (num_status_desc > md_avail_desc_ring_space(&chan->dma_ctx->dma_dev, | |
1007 | is_astep, chan->chan, (uint32_t)chan->next_write_index, num_status_desc)) { | |
1008 | cpu_relax(); | |
1009 | if (time_after(jiffies,ts + DMA_TO)) { | |
1010 | printk(KERN_ERR "%s %d TO chan 0x%x\n", __func__, __LINE__, chan->ch_num); | |
1011 | return -EBUSY; | |
1012 | } | |
1013 | } | |
1014 | ||
1015 | chan->intr_ring.comp_cb_array[intr_ring_index] = NULL; | |
1016 | ||
1017 | incr_head(&chan->intr_ring.ring); | |
1018 | #ifdef CONFIG_MK1OM | |
1019 | md_mic_dma_prep_status_desc(&chan->desc_ring[chan->next_write_index], | |
1020 | intr_ring_index, | |
1021 | chan->intr_ring.ring.tail_phys, | |
1022 | false); | |
1023 | chan->next_write_index = incr_rb_index((int)chan->next_write_index, | |
1024 | chan->chan->num_desc_in_ring); | |
1025 | #endif | |
1026 | md_mic_dma_prep_status_desc(&chan->desc_ring[chan->next_write_index], | |
1027 | intr_ring_index, | |
1028 | chan->intr_ring.ring.tail_phys, | |
1029 | true); | |
1030 | chan->next_write_index = incr_rb_index((int)chan->next_write_index, | |
1031 | chan->chan->num_desc_in_ring); | |
1032 | ||
1033 | md_mic_dma_chan_write_head(&chan->dma_ctx->dma_dev, chan->chan, (uint32_t)chan->next_write_index); | |
1034 | return intr_ring_index; | |
1035 | } | |
1036 | EXPORT_SYMBOL(program_dma_mark); | |
1037 | ||
1038 | /* | |
1039 | * is_current_dma_mark: Check if the dma mark provided is the current DMA mark. | |
1040 | * @chan - DMA channel | |
1041 | * @mark - DMA mark | |
1042 | * | |
1043 | * Return true on success and false on failure. | |
1044 | */ | |
1045 | bool is_current_dma_mark(struct dma_channel *chan, int mark) | |
1046 | { | |
1047 | return (get_dma_mark(chan) == mark); | |
1048 | } | |
1049 | EXPORT_SYMBOL(is_current_dma_mark); | |
1050 | ||
1051 | /* | |
1052 | * is_dma_mark_processed: Check if the dma mark provided has been processed. | |
1053 | * @chan - DMA channel | |
1054 | * @mark - DMA mark | |
1055 | * | |
1056 | * Return true on success and false on failure. | |
1057 | */ | |
1058 | bool is_dma_mark_processed(struct dma_channel *chan, int mark) | |
1059 | { | |
1060 | return is_entry_processed(&chan->intr_ring.ring, mark); | |
1061 | } | |
1062 | EXPORT_SYMBOL(is_dma_mark_processed); | |
1063 | ||
1064 | /* | |
1065 | * dma_mark_wait: | |
1066 | * @chan - DMA channel | |
1067 | * @mark - DMA mark | |
1068 | * @is_interruptible - Use wait_event_interruptible() or not. | |
1069 | * | |
1070 | * Wait for the dma mark to complete. | |
1071 | * Return 0 on success and appropriate error value on error. | |
1072 | */ | |
1073 | int dma_mark_wait(struct dma_channel *chan, int mark, bool is_interruptible) | |
1074 | { | |
1075 | int err = 0; | |
1076 | uint32_t prev_tail = 0, new_tail; | |
1077 | uint32_t count = 0; | |
1078 | ||
1079 | if (chan) { | |
1080 | might_sleep(); | |
1081 | __retry: | |
1082 | if (is_interruptible) | |
1083 | err = wait_event_interruptible_timeout( | |
1084 | chan->intr_wq, | |
1085 | is_dma_mark_processed(chan, mark), | |
1086 | DMA_TO); | |
1087 | else | |
1088 | err = wait_event_timeout(chan->intr_wq, | |
1089 | is_dma_mark_processed(chan, mark), DMA_TO); | |
1090 | ||
1091 | if (!err) { // 0 is timeout | |
1092 | new_tail = get_dma_tail_pointer(chan); | |
1093 | if ((count <= DMA_FENCE_TIMEOUT_CNT) && | |
1094 | (!count || new_tail != prev_tail)) { // For performance, prev_tail is not read at the begining | |
1095 | prev_tail = new_tail; | |
1096 | count++; | |
1097 | pr_debug("DMA fence wating is still ongoing, waiting for %d seconds\n", DMA_TO/HZ *count); | |
1098 | goto __retry; | |
1099 | } else { | |
1100 | printk(KERN_ERR "%s %d TO chan 0x%x\n", __func__, __LINE__, chan->ch_num); | |
1101 | err = -EBUSY; | |
1102 | } | |
1103 | } | |
1104 | if (err > 0) | |
1105 | err = 0; | |
1106 | } | |
1107 | return err; | |
1108 | } | |
1109 | EXPORT_SYMBOL(dma_mark_wait); | |
1110 | ||
1111 | /* | |
1112 | * drain_dma_poll - Drain all outstanding DMA operations for a particular | |
1113 | * DMA channel via polling. | |
1114 | * @chan - DMA channel | |
1115 | * Return 0 on success and -errno on error. | |
1116 | */ | |
1117 | int drain_dma_poll(struct dma_channel *chan) | |
1118 | { | |
1119 | int cookie, err; | |
1120 | unsigned long ts; | |
1121 | uint32_t prev_tail = 0, new_tail, count = 0; | |
1122 | if (chan) { | |
1123 | if ((err = request_dma_channel(chan))) | |
1124 | goto error; | |
1125 | if ((cookie = do_dma(chan, | |
1126 | DO_DMA_POLLING, 0, 0, 0, NULL)) < 0) { | |
1127 | err = cookie; | |
1128 | free_dma_channel(chan); | |
1129 | goto error; | |
1130 | } | |
1131 | free_dma_channel(chan); | |
1132 | ts = jiffies; | |
1133 | while (1 != poll_dma_completion(cookie, chan)) { | |
1134 | cpu_relax(); | |
1135 | if (time_after(jiffies,ts + DMA_TO)) { | |
1136 | new_tail = get_dma_tail_pointer(chan); | |
1137 | if ((!count || new_tail != prev_tail) && (count <= DMA_FENCE_TIMEOUT_CNT)) { | |
1138 | prev_tail = new_tail; | |
1139 | ts = jiffies; | |
1140 | count++; | |
1141 | pr_debug("polling DMA is still ongoing, wating for %d seconds\n", DMA_TO/HZ * count); | |
1142 | } else { | |
1143 | err = -EBUSY; | |
1144 | break; | |
1145 | } | |
1146 | } | |
1147 | } | |
1148 | error: | |
1149 | if (err) | |
1150 | printk(KERN_ERR "%s %d err %d\n", __func__, __LINE__, err); | |
1151 | } else { | |
1152 | err = -EINVAL; | |
1153 | } | |
1154 | return err; | |
1155 | } | |
1156 | EXPORT_SYMBOL(drain_dma_poll); | |
1157 | ||
1158 | /* | |
1159 | * drain_dma_intr - Drain all outstanding DMA operations for a particular | |
1160 | * DMA channel via interrupt based blocking wait. | |
1161 | * @chan - DMA channel | |
1162 | * Return 0 on success and -errno on error. | |
1163 | */ | |
1164 | int drain_dma_intr(struct dma_channel *chan) | |
1165 | { | |
1166 | int cookie, err; | |
1167 | ||
1168 | if (chan) { | |
1169 | if ((err = request_dma_channel(chan))) | |
1170 | goto error; | |
1171 | if ((cookie = program_dma_mark(chan)) < 0) { | |
1172 | err = cookie; | |
1173 | free_dma_channel(chan); | |
1174 | goto error; | |
1175 | } | |
1176 | free_dma_channel(chan); | |
1177 | err = dma_mark_wait(chan, cookie, false); | |
1178 | error: | |
1179 | if (err) | |
1180 | printk(KERN_ERR "%s %d err %d\n", __func__, __LINE__, err); | |
1181 | } else { | |
1182 | err = -EINVAL; | |
1183 | } | |
1184 | return err; | |
1185 | } | |
1186 | EXPORT_SYMBOL(drain_dma_intr); | |
1187 | ||
1188 | /* | |
1189 | * drain_dma_global - Drain all outstanding DMA operations for | |
1190 | * all online DMA channel. | |
1191 | * Return none | |
1192 | */ | |
1193 | int drain_dma_global(mic_dma_handle_t dma_handle) | |
1194 | { | |
1195 | int i, err = -EINVAL; | |
1196 | struct dma_channel *chan; | |
1197 | struct mic_dma_ctx_t *dma_ctx = (struct mic_dma_ctx_t *)dma_handle; | |
1198 | ||
1199 | if (!dma_ctx) | |
1200 | return err; | |
1201 | ||
1202 | might_sleep(); | |
1203 | for (i = 0 ; i < MAX_NUM_DMA_CHAN; i++) { | |
1204 | chan = &dma_ctx->dma_channels[i]; | |
1205 | if (chan->desc_ring == NULL) | |
1206 | continue; | |
1207 | if ((err = drain_dma_intr(chan))) | |
1208 | break; | |
1209 | } | |
1210 | return err; | |
1211 | } | |
1212 | EXPORT_SYMBOL(drain_dma_global); | |
1213 | ||
1214 | #ifdef _MIC_SCIF_ | |
1215 | /* | |
1216 | * dma_suspend: DMA tasks before transition to low power state. | |
1217 | * @dma_handle: Handle for a DMA driver context. | |
1218 | * | |
1219 | * Perform the following tasks before the device transitions | |
1220 | * to a low power state: | |
1221 | * 1) Store away the DMA descriptor ring physical address base for | |
1222 | * all DMA channels (both host/uOS owned) since the value would be | |
1223 | * required to reinitialize the DMA channels upon transition from | |
1224 | * low power to active state. | |
1225 | * | |
1226 | * Return: none | |
1227 | * Notes: Invoked only on MIC. | |
1228 | */ | |
1229 | void dma_suspend(mic_dma_handle_t dma_handle) | |
1230 | { | |
1231 | int i; | |
1232 | struct dma_channel *ch; | |
1233 | struct mic_dma_ctx_t *dma_ctx = (struct mic_dma_ctx_t *)dma_handle; | |
1234 | struct mic_dma_device *dma_dev = &dma_ctx->dma_dev; | |
1235 | ||
1236 | for (i = 0; i < MAX_NUM_DMA_CHAN; i++) { | |
1237 | ch = &dma_ctx->dma_channels[i]; | |
1238 | ch->desc_ring_phys = | |
1239 | md_mic_dma_chan_get_desc_ring_phys(dma_dev, ch->chan); | |
1240 | ch->chan->dstat_wb_phys = | |
1241 | md_mic_dma_chan_get_dstatwb_phys(dma_dev, ch->chan); | |
1242 | } | |
1243 | } | |
1244 | EXPORT_SYMBOL(dma_suspend); | |
1245 | ||
1246 | /* | |
1247 | * dma_resume: DMA tasks after wake up from low power state. | |
1248 | * @dma_handle: Handle for a DMA driver context. | |
1249 | * | |
1250 | * Performs the following tasks before the device transitions | |
1251 | * from a low power state to active state: | |
1252 | * 1) As a test, reset the value in DMA configuration register. | |
1253 | * 2) Reset the next_write_index for the DMA descriptor ring to 0 | |
1254 | * since the DMA channel will be reset shortly. | |
1255 | * 3) Reinitialize the DMA MD layer for the channel. | |
1256 | * | |
1257 | * Return: none | |
1258 | * Notes: | |
1259 | * Notes: Invoked only on MIC. | |
1260 | */ | |
1261 | void dma_resume(mic_dma_handle_t dma_handle) | |
1262 | { | |
1263 | int i; | |
1264 | struct dma_channel *ch; | |
1265 | struct mic_dma_ctx_t *dma_ctx = (struct mic_dma_ctx_t *)dma_handle; | |
1266 | struct mic_dma_device *dma_dev = &dma_ctx->dma_dev; | |
1267 | ||
1268 | /* TODO: Remove test write to SBOX_DCR */ | |
1269 | mic_sbox_write_mmio(dma_dev->mm_sbox, SBOX_DCR, 0); | |
1270 | for (i = 0; i < MAX_NUM_DMA_CHAN; i++) { | |
1271 | ch = &dma_ctx->dma_channels[i]; | |
1272 | ch->next_write_index = 0; | |
1273 | md_mic_dma_chan_init_attr(dma_dev, ch->chan); | |
1274 | md_mic_dma_chan_setup(dma_ctx, ch); | |
1275 | } | |
1276 | } | |
1277 | EXPORT_SYMBOL(dma_resume); | |
1278 | ||
1279 | #else | |
1280 | ||
1281 | /* | |
1282 | * dma_prep_suspend: DMA tasks required on host before a device can transition | |
1283 | * to a low power state. | |
1284 | * @dma_handle: Handle for a DMA driver context. | |
1285 | * | |
1286 | * Performs the following tasks on the host before the device can be allowed | |
1287 | * to transiti to a low power state. | |
1288 | * 1) Reset the next_Write_index for the DMA descriptor ring to 0 | |
1289 | * since the DMA channel will be reset shortly. This is required primarily | |
1290 | * for Host owned DMA channels since MIC does not have access to this | |
1291 | * information. | |
1292 | * Return: none | |
1293 | * Invoked only on Host. | |
1294 | */ | |
1295 | void dma_prep_suspend(mic_dma_handle_t dma_handle) | |
1296 | { | |
1297 | int i; | |
1298 | struct dma_channel *ch; | |
1299 | struct mic_dma_ctx_t *dma_ctx = (struct mic_dma_ctx_t *)dma_handle; | |
1300 | ||
1301 | for (i = 0; i < MAX_NUM_DMA_CHAN; i++) { | |
1302 | ch = &dma_ctx->dma_channels[i]; | |
1303 | ch->next_write_index = 0; | |
1304 | } | |
1305 | } | |
1306 | EXPORT_SYMBOL(dma_prep_suspend); | |
1307 | #endif | |
1308 | ||
1309 | #ifdef CONFIG_PAGE_CACHE_DMA | |
1310 | #ifdef _MIC_SCIF_ | |
1311 | static const struct dma_operations dma_operations_fast_copy = { | |
1312 | .do_dma = do_dma, | |
1313 | .poll_dma_completion = poll_dma_completion, | |
1314 | .free_dma_channel = free_dma_channel, | |
1315 | .open_dma_device = open_dma_device, | |
1316 | .close_dma_device = close_dma_device, | |
1317 | .allocate_dma_channel = allocate_dma_channel, | |
1318 | .program_descriptors = program_memcpy_descriptors, | |
1319 | .do_dma_polling = DO_DMA_POLLING, | |
1320 | }; | |
1321 | ||
1322 | static const struct file_dma fdma_callback = { | |
1323 | .dmaops = &dma_operations_fast_copy, | |
1324 | }; | |
1325 | #endif | |
1326 | #endif | |
1327 | ||
1328 | #ifdef _MIC_SCIF_ | |
1329 | static int | |
1330 | #else | |
1331 | int | |
1332 | #endif | |
1333 | mic_dma_init(void) | |
1334 | { | |
1335 | int i; | |
1336 | ||
1337 | for (i = 0; i < MAX_BOARD_SUPPORTED; i++) | |
1338 | mutex_init (&lock_dma_dev_init[i]); | |
1339 | #ifdef CONFIG_PAGE_CACHE_DMA | |
1340 | #ifdef _MIC_SCIF_ | |
1341 | register_dma_for_fast_copy(&fdma_callback); | |
1342 | #endif | |
1343 | #endif | |
1344 | return 0; | |
1345 | } | |
1346 | ||
1347 | #ifdef _MIC_SCIF_ | |
1348 | static void mic_dma_uninit(void) | |
1349 | { | |
1350 | #ifdef CONFIG_PAGE_CACHE_DMA | |
1351 | unregister_dma_for_fast_copy(); | |
1352 | #endif | |
1353 | } | |
1354 | ||
1355 | module_init(mic_dma_init); | |
1356 | module_exit(mic_dma_uninit); | |
1357 | #endif | |
1358 | ||
1359 | #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)) | |
1360 | static int | |
1361 | mic_dma_proc_ring_show(struct seq_file *m, void *data) | |
1362 | { | |
1363 | struct mic_dma_ctx_t *dma_ctx = m->private; | |
1364 | mic_ctx_t *mic_ctx = get_per_dev_ctx(dma_ctx->device_num - 1); | |
1365 | int i, err; | |
1366 | struct compl_buf_ring *ring; | |
1367 | ||
1368 | if ((err = micpm_get_reference(mic_ctx, true))) { | |
1369 | printk(KERN_ERR "%s %d: unable to get micpm reference: %d\n", | |
1370 | __func__, __LINE__, err); | |
1371 | return err; | |
1372 | } | |
1373 | ||
1374 | seq_printf(m, "Intr rings\n"); | |
1375 | seq_printf(m, "%-10s%-12s%-12s%-12s%-25s%-18s%-25s\n", | |
1376 | "Chan", "Head", "Tail", "Size", "Tail loc", "Actual tail", "In Use"); | |
1377 | for (i = first_dma_chan(); i <= last_dma_chan(); i++) { | |
1378 | ring = &dma_ctx->dma_channels[i].intr_ring.ring; | |
1379 | seq_printf(m, "%-#10x%-#12x%-#12x%-#12x%-#25llx%-#18x%-#18x\n", | |
1380 | i, ring->head, ring->tail, ring->size, | |
1381 | ring->tail_location, *(int*)ring->tail_location, | |
1382 | atomic_read(&dma_ctx->dma_channels[i].flags)); | |
1383 | } | |
1384 | seq_printf(m, "Poll rings\n"); | |
1385 | seq_printf(m, "%-10s%-12s%-12s%-12s%-25s%-18s\n", | |
1386 | "Chan", "Head", "Tail", "Size", "Tail loc", "Actual tail"); | |
1387 | for (i = first_dma_chan(); i <= last_dma_chan(); i++) { | |
1388 | ring = &dma_ctx->dma_channels[i].poll_ring; | |
1389 | seq_printf(m, "%-#10x%-#12x%-#12x%-#12x%-#25llx%-#18x\n", | |
1390 | i, ring->head, ring->tail, ring->size, | |
1391 | ring->tail_location, *(int*)ring->tail_location); | |
1392 | } | |
1393 | seq_printf(m, "Next_Write_Index\n"); | |
1394 | seq_printf(m, "%-10s%-12s\n", "Chan", "Next_Write_Index"); | |
1395 | for (i = 0; i < MAX_NUM_DMA_CHAN; i++) { | |
1396 | seq_printf(m, "%-#10x%-#12llx\n", | |
1397 | i, dma_ctx->dma_channels[i].next_write_index); | |
1398 | } | |
1399 | micpm_put_reference(mic_ctx); | |
1400 | return 0; | |
1401 | } | |
1402 | ||
1403 | static int | |
1404 | mic_dma_proc_ring_open(struct inode *inode, struct file *file) | |
1405 | { | |
1406 | return single_open(file, mic_dma_proc_ring_show, PDE_DATA(inode)); | |
1407 | } | |
1408 | ||
1409 | static int | |
1410 | mic_dma_proc_reg_show(struct seq_file *m, void *data) | |
1411 | { | |
1412 | int i, j, chan_num, size, dtpr, err; | |
1413 | struct mic_dma_ctx_t *dma_ctx = m->private; | |
1414 | mic_ctx_t *mic_ctx = get_per_dev_ctx(dma_ctx->device_num - 1); | |
1415 | struct mic_dma_device *dma_dev = &dma_ctx->dma_dev; | |
1416 | struct dma_channel *curr_chan; | |
1417 | union md_mic_dma_desc desc; | |
1418 | ||
1419 | if ((err = micpm_get_reference(mic_ctx, true))) { | |
1420 | printk(KERN_ERR "%s %d: unable to get micpm reference: %d\n", | |
1421 | __func__, __LINE__, err); | |
1422 | return err; | |
1423 | } | |
1424 | ||
1425 | seq_printf(m, "========================================" | |
1426 | "=======================================\n"); | |
1427 | seq_printf(m, "SBOX_DCR: %#x\n", | |
1428 | mic_sbox_read_mmio(dma_dev->mm_sbox, SBOX_DCR)); | |
1429 | seq_printf(m, "DMA Channel Registers\n"); | |
1430 | seq_printf(m, "========================================" | |
1431 | "=======================================\n"); | |
1432 | seq_printf(m, "%-10s| %-10s %-10s %-10s %-10s %-10s %-10s" | |
1433 | #ifdef CONFIG_MK1OM | |
1434 | " %-10s %-11s %-14s %-10s" | |
1435 | #endif | |
1436 | "\n", "Channel", "DCAR", "DTPR", "DHPR", | |
1437 | "DRAR_HI", "DRAR_LO", | |
1438 | #ifdef CONFIG_MK1OM | |
1439 | "DSTATWB_LO", "DSTATWB_HI", "DSTAT_CHERR", "DSTAT_CHERRMSK", | |
1440 | #endif | |
1441 | "DSTAT"); | |
1442 | seq_printf(m, "========================================" | |
1443 | "=======================================\n"); | |
1444 | ||
1445 | #ifdef _MIC_SCIF_ | |
1446 | for (i = 0; i < MAX_NUM_DMA_CHAN; i++) { | |
1447 | #else | |
1448 | for (i = first_dma_chan(); i <= last_dma_chan(); i++) { | |
1449 | #endif | |
1450 | curr_chan = &dma_ctx->dma_channels[i]; | |
1451 | chan_num = curr_chan->ch_num; | |
1452 | seq_printf(m, "%-10i| %-#10x %-#10x %-#10x %-#10x" | |
1453 | " %-#10x" | |
1454 | #ifdef CONFIG_MK1OM | |
1455 | " %-#10x %-#11x %-#10x %-#14x" | |
1456 | #endif | |
1457 | " %-#10x\n", chan_num, | |
1458 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCAR), | |
1459 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DTPR), | |
1460 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DHPR), | |
1461 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DRAR_HI), | |
1462 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DRAR_LO), | |
1463 | #ifdef CONFIG_MK1OM | |
1464 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTATWB_LO), | |
1465 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTATWB_HI), | |
1466 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCHERR), | |
1467 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCHERRMSK), | |
1468 | #endif | |
1469 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTAT)); | |
1470 | } | |
1471 | ||
1472 | seq_printf(m, "\nDMA Channel Descriptor Rings\n"); | |
1473 | seq_printf(m, "========================================" | |
1474 | "=======================================\n"); | |
1475 | ||
1476 | for (i = first_dma_chan(); i <= last_dma_chan(); i++) { | |
1477 | curr_chan = &dma_ctx->dma_channels[i]; | |
1478 | chan_num = curr_chan->ch_num; | |
1479 | dtpr = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DTPR); | |
1480 | seq_printf(m, "Channel %i: [", chan_num); | |
1481 | size = ((int) md_mic_dma_read_mmio(dma_dev, chan_num, REG_DHPR) | |
1482 | - dtpr) % curr_chan->chan->num_desc_in_ring; | |
1483 | /* | |
1484 | * In KNC B0, empty condition is tail = head -1 | |
1485 | */ | |
1486 | if (mic_hw_family(dma_ctx->device_num) == FAMILY_KNC && | |
1487 | mic_hw_stepping(dma_ctx->device_num) >= KNC_B0_STEP) | |
1488 | size -= 1; | |
1489 | ||
1490 | for (j = 0; j < size; j++) { | |
1491 | desc = curr_chan->desc_ring[(j+dtpr) % | |
1492 | curr_chan->chan->num_desc_in_ring]; | |
1493 | ||
1494 | switch (desc.desc.nop.type){ | |
1495 | case NOP: | |
1496 | seq_printf(m," {Type: NOP, 0x%#llx" | |
1497 | " %#llx} ", desc.qwords.qw0, | |
1498 | desc.qwords.qw1); | |
1499 | case MEMCOPY: | |
1500 | seq_printf(m," {Type: MEMCOPY, SAP:" | |
1501 | " 0x%#llx, DAP: %#llx, length: %#llx} ", | |
1502 | (uint64_t) desc.desc.memcopy.sap, | |
1503 | (uint64_t) desc.desc.memcopy.dap, | |
1504 | (uint64_t) desc.desc.memcopy.length); | |
1505 | break; | |
1506 | case STATUS: | |
1507 | seq_printf(m," {Type: STATUS, data:" | |
1508 | " 0x%#llx, DAP: %#llx, intr: %lli} ", | |
1509 | (uint64_t) desc.desc.status.data, | |
1510 | (uint64_t) desc.desc.status.dap, | |
1511 | (uint64_t) desc.desc.status.intr); | |
1512 | break; | |
1513 | case GENERAL: | |
1514 | seq_printf(m," {Type: GENERAL, " | |
1515 | "DAP: %#llx, dword: %#llx} ", | |
1516 | (uint64_t) desc.desc.general.dap, | |
1517 | (uint64_t) desc.desc.general.data); | |
1518 | break; | |
1519 | case KEYNONCECNT: | |
1520 | seq_printf(m," {Type: KEYNONCECNT, sel: " | |
1521 | "%lli, h: %lli, index: %lli, cs: %lli," | |
1522 | " value: %#llx} ", | |
1523 | (uint64_t) desc.desc.keynoncecnt.sel, | |
1524 | (uint64_t) desc.desc.keynoncecnt.h, | |
1525 | (uint64_t) desc.desc.keynoncecnt.index, | |
1526 | (uint64_t) desc.desc.keynoncecnt.cs, | |
1527 | (uint64_t) desc.desc.keynoncecnt.data); | |
1528 | break; | |
1529 | case KEY: | |
1530 | seq_printf(m," {Type: KEY, dest_ind" | |
1531 | "ex: %lli, ski: %lli, skap: %#llx ", | |
1532 | (uint64_t) desc.desc.key.di, | |
1533 | (uint64_t) desc.desc.key.ski, | |
1534 | (uint64_t) desc.desc.key.skap); | |
1535 | break; | |
1536 | default: | |
1537 | seq_printf(m," {Uknown Type=%lli ," | |
1538 | "%#llx %#llx} ",(uint64_t) desc.desc.nop.type, | |
1539 | (uint64_t) desc.qwords.qw0, | |
1540 | (uint64_t) desc.qwords.qw1); | |
1541 | } | |
1542 | } | |
1543 | seq_printf(m, "]\n"); | |
1544 | if (mic_hw_family(dma_ctx->device_num) == FAMILY_KNC && | |
1545 | mic_hw_stepping(dma_ctx->device_num) >= KNC_B0_STEP && | |
1546 | curr_chan->chan->dstat_wb_loc) | |
1547 | seq_printf(m, "DSTAT_WB = 0x%x\n", | |
1548 | *((uint32_t*)curr_chan->chan->dstat_wb_loc)); | |
1549 | } | |
1550 | micpm_put_reference(mic_ctx); | |
1551 | ||
1552 | return 0; | |
1553 | } | |
1554 | ||
1555 | static int | |
1556 | mic_dma_proc_reg_open(struct inode *inode, struct file *file) | |
1557 | { | |
1558 | return single_open(file, mic_dma_proc_reg_show, PDE_DATA(inode)); | |
1559 | } | |
1560 | ||
1561 | struct file_operations micdma_ring_fops = { | |
1562 | .open = mic_dma_proc_ring_open, | |
1563 | .read = seq_read, | |
1564 | .llseek = seq_lseek, | |
1565 | .release = single_release, | |
1566 | }; | |
1567 | ||
1568 | struct file_operations micdma_reg_fops = { | |
1569 | .open = mic_dma_proc_reg_open, | |
1570 | .read = seq_read, | |
1571 | .llseek = seq_lseek, | |
1572 | .release = single_release, | |
1573 | }; | |
1574 | ||
1575 | static void | |
1576 | mic_dma_proc_init(struct mic_dma_ctx_t *dma_ctx) | |
1577 | { | |
1578 | char name[64]; | |
1579 | ||
1580 | snprintf(name, 63, "%s%d", proc_dma_ring, dma_ctx->device_num); | |
1581 | if (!proc_create_data(name, S_IFREG | S_IRUGO, NULL, &micdma_ring_fops, dma_ctx)) | |
1582 | printk("micdma: unable to register /proc/%s\n", name); | |
1583 | ||
1584 | snprintf(name, 63, "%s%d", proc_dma_reg, dma_ctx->device_num); | |
1585 | if (!proc_create_data(name, S_IFREG | S_IRUGO, NULL, &micdma_reg_fops, dma_ctx)) | |
1586 | printk("micdma: unable to register /proc/%s\n", name); | |
1587 | ||
1588 | } | |
1589 | #else // LINUX VERSION | |
1590 | static int | |
1591 | mic_dma_proc_read_fn(char *buf, char **start, off_t offset, int count, int *eof, void *data) | |
1592 | { | |
1593 | struct mic_dma_ctx_t *dma_ctx = data; | |
1594 | int i, len = 0; | |
1595 | struct compl_buf_ring *ring; | |
1596 | ||
1597 | len += sprintf(buf + len, "Intr rings\n"); | |
1598 | len += sprintf(buf + len, "%-10s%-12s%-12s%-12s%-25s%-18s%-25s\n", | |
1599 | "Chan", "Head", "Tail", "Size", "Tail loc", "Actual tail", "In Use"); | |
1600 | for (i = first_dma_chan(); i <= last_dma_chan(); i++) { | |
1601 | ring = &dma_ctx->dma_channels[i].intr_ring.ring; | |
1602 | len += sprintf(buf + len, "%-#10x%-#12x%-#12x%-#12x%-#25llx%-#18x%-#18x\n", | |
1603 | i, ring->head, ring->tail, ring->size, | |
1604 | ring->tail_location, *(int*)ring->tail_location, | |
1605 | atomic_read(&dma_ctx->dma_channels[i].flags)); | |
1606 | } | |
1607 | len += sprintf(buf + len, "Poll rings\n"); | |
1608 | len += sprintf(buf + len, "%-10s%-12s%-12s%-12s%-25s%-18s\n", | |
1609 | "Chan", "Head", "Tail", "Size", "Tail loc", "Actual tail"); | |
1610 | for (i = first_dma_chan(); i <= last_dma_chan(); i++) { | |
1611 | ring = &dma_ctx->dma_channels[i].poll_ring; | |
1612 | len += sprintf(buf + len, "%-#10x%-#12x%-#12x%-#12x%-#25llx%-#18x\n", | |
1613 | i, ring->head, ring->tail, ring->size, | |
1614 | ring->tail_location, *(int*)ring->tail_location); | |
1615 | } | |
1616 | len += sprintf(buf + len, "Next_Write_Index\n"); | |
1617 | len += sprintf(buf + len, "%-10s%-12s\n", "Chan", "Next_Write_Index"); | |
1618 | for (i = 0; i < MAX_NUM_DMA_CHAN; i++) { | |
1619 | len += sprintf(buf + len, "%-#10x%-#12llx\n", | |
1620 | i, dma_ctx->dma_channels[i].next_write_index); | |
1621 | } | |
1622 | return len; | |
1623 | } | |
1624 | ||
1625 | static int | |
1626 | mic_dma_proc_read_registers_fn(char *buf, char **start, off_t offset, int count, | |
1627 | int *eof, void *data) | |
1628 | { | |
1629 | int i, j, chan_num, size, dtpr, len = 0; | |
1630 | struct mic_dma_ctx_t *dma_ctx = data; | |
1631 | struct mic_dma_device *dma_dev = &dma_ctx->dma_dev; | |
1632 | struct dma_channel *curr_chan; | |
1633 | union md_mic_dma_desc desc; | |
1634 | ||
1635 | len += sprintf(buf + len, "========================================" | |
1636 | "=======================================\n"); | |
1637 | len += sprintf(buf + len, "SBOX_DCR: %#x\n", | |
1638 | mic_sbox_read_mmio(dma_dev->mm_sbox, SBOX_DCR)); | |
1639 | len += sprintf(buf + len, "DMA Channel Registers\n"); | |
1640 | len += sprintf(buf + len, "========================================" | |
1641 | "=======================================\n"); | |
1642 | len += sprintf(buf + len, "%-10s| %-10s %-10s %-10s %-10s %-10s %-10s" | |
1643 | #ifdef CONFIG_MK1OM | |
1644 | " %-10s %-11s %-14s %-10s" | |
1645 | #endif | |
1646 | "\n", "Channel", "DCAR", "DTPR", "DHPR", | |
1647 | "DRAR_HI", "DRAR_LO", | |
1648 | #ifdef CONFIG_MK1OM | |
1649 | "DSTATWB_LO", "DSTATWB_HI", "DSTAT_CHERR", "DSTAT_CHERRMSK", | |
1650 | #endif | |
1651 | "DSTAT"); | |
1652 | len += sprintf(buf + len, "========================================" | |
1653 | "=======================================\n"); | |
1654 | ||
1655 | #ifdef _MIC_SCIF_ | |
1656 | for (i = 0; i < MAX_NUM_DMA_CHAN; i++) { | |
1657 | #else | |
1658 | for (i = first_dma_chan(); i <= last_dma_chan(); i++) { | |
1659 | #endif | |
1660 | curr_chan = &dma_ctx->dma_channels[i]; | |
1661 | chan_num = curr_chan->ch_num; | |
1662 | len += sprintf(buf + len, "%-10i| %-#10x %-#10x %-#10x %-#10x" | |
1663 | " %-#10x" | |
1664 | #ifdef CONFIG_MK1OM | |
1665 | " %-#10x %-#11x %-#10x %-#14x" | |
1666 | #endif | |
1667 | " %-#10x\n", chan_num, | |
1668 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCAR), | |
1669 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DTPR), | |
1670 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DHPR), | |
1671 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DRAR_HI), | |
1672 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DRAR_LO), | |
1673 | #ifdef CONFIG_MK1OM | |
1674 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTATWB_LO), | |
1675 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTATWB_HI), | |
1676 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCHERR), | |
1677 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCHERRMSK), | |
1678 | #endif | |
1679 | md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTAT)); | |
1680 | } | |
1681 | ||
1682 | len += sprintf(buf + len, "\nDMA Channel Descriptor Rings\n"); | |
1683 | len += sprintf(buf + len, "========================================" | |
1684 | "=======================================\n"); | |
1685 | ||
1686 | for (i = first_dma_chan(); i <= last_dma_chan(); i++) { | |
1687 | curr_chan = &dma_ctx->dma_channels[i]; | |
1688 | chan_num = curr_chan->ch_num; | |
1689 | dtpr = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DTPR); | |
1690 | len += sprintf(buf + len, "Channel %i: [", chan_num); | |
1691 | size = ((int) md_mic_dma_read_mmio(dma_dev, chan_num, REG_DHPR) | |
1692 | - dtpr) % curr_chan->chan->num_desc_in_ring; | |
1693 | /* | |
1694 | * In KNC B0, empty condition is tail = head -1 | |
1695 | */ | |
1696 | if (mic_hw_family(dma_ctx->device_num) == FAMILY_KNC && | |
1697 | mic_hw_stepping(dma_ctx->device_num) >= KNC_B0_STEP) | |
1698 | size -= 1; | |
1699 | ||
1700 | for (j = 0; j < size; j++) { | |
1701 | desc = curr_chan->desc_ring[(j+dtpr) % | |
1702 | curr_chan->chan->num_desc_in_ring]; | |
1703 | ||
1704 | switch (desc.desc.nop.type){ | |
1705 | case NOP: | |
1706 | len += sprintf(buf + len," {Type: NOP, 0x%#llx" | |
1707 | " %#llx} ", desc.qwords.qw0, | |
1708 | desc.qwords.qw1); | |
1709 | case MEMCOPY: | |
1710 | len += sprintf(buf + len," {Type: MEMCOPY, SAP:" | |
1711 | " 0x%#llx, DAP: %#llx, length: %#llx} ", | |
1712 | (uint64_t) desc.desc.memcopy.sap, | |
1713 | (uint64_t) desc.desc.memcopy.dap, | |
1714 | (uint64_t) desc.desc.memcopy.length); | |
1715 | break; | |
1716 | case STATUS: | |
1717 | len += sprintf(buf + len," {Type: STATUS, data:" | |
1718 | " 0x%#llx, DAP: %#llx, intr: %lli} ", | |
1719 | (uint64_t) desc.desc.status.data, | |
1720 | (uint64_t) desc.desc.status.dap, | |
1721 | (uint64_t) desc.desc.status.intr); | |
1722 | break; | |
1723 | case GENERAL: | |
1724 | len += sprintf(buf + len," {Type: GENERAL, " | |
1725 | "DAP: %#llx, dword: %#llx} ", | |
1726 | (uint64_t) desc.desc.general.dap, | |
1727 | (uint64_t) desc.desc.general.data); | |
1728 | break; | |
1729 | case KEYNONCECNT: | |
1730 | len += sprintf(buf + len," {Type: KEYNONCECNT, sel: " | |
1731 | "%lli, h: %lli, index: %lli, cs: %lli," | |
1732 | " value: %#llx} ", | |
1733 | (uint64_t) desc.desc.keynoncecnt.sel, | |
1734 | (uint64_t) desc.desc.keynoncecnt.h, | |
1735 | (uint64_t) desc.desc.keynoncecnt.index, | |
1736 | (uint64_t) desc.desc.keynoncecnt.cs, | |
1737 | (uint64_t) desc.desc.keynoncecnt.data); | |
1738 | break; | |
1739 | case KEY: | |
1740 | len += sprintf(buf + len," {Type: KEY, dest_ind" | |
1741 | "ex: %lli, ski: %lli, skap: %#llx ", | |
1742 | (uint64_t) desc.desc.key.di, | |
1743 | (uint64_t) desc.desc.key.ski, | |
1744 | (uint64_t) desc.desc.key.skap); | |
1745 | break; | |
1746 | default: | |
1747 | len += sprintf(buf + len," {Uknown Type=%lli ," | |
1748 | "%#llx %#llx} ",(uint64_t) desc.desc.nop.type, | |
1749 | (uint64_t) desc.qwords.qw0, | |
1750 | (uint64_t) desc.qwords.qw1); | |
1751 | } | |
1752 | } | |
1753 | len += sprintf(buf + len, "]\n"); | |
1754 | if (mic_hw_family(dma_ctx->device_num) == FAMILY_KNC && | |
1755 | mic_hw_stepping(dma_ctx->device_num) >= KNC_B0_STEP && | |
1756 | curr_chan->chan->dstat_wb_loc) | |
1757 | len += sprintf(buf + len, "DSTAT_WB = 0x%x\n", | |
1758 | *((uint32_t*)curr_chan->chan->dstat_wb_loc)); | |
1759 | } | |
1760 | return len; | |
1761 | } | |
1762 | ||
1763 | static void | |
1764 | mic_dma_proc_init(struct mic_dma_ctx_t *dma_ctx) | |
1765 | { | |
1766 | struct proc_dir_entry *dma_proc; | |
1767 | char name[64]; | |
1768 | ||
1769 | snprintf(name, 63, "%s%d", proc_dma_ring, dma_ctx->device_num); | |
1770 | if ((dma_proc = create_proc_entry(name, S_IFREG | S_IRUGO, NULL)) != NULL) { | |
1771 | dma_proc->read_proc = mic_dma_proc_read_fn; | |
1772 | dma_proc->data = dma_ctx; | |
1773 | } | |
1774 | snprintf(name, 63, "%s%d", proc_dma_reg, dma_ctx->device_num); | |
1775 | if ((dma_proc = create_proc_entry(name, S_IFREG | S_IRUGO, NULL)) != NULL) { | |
1776 | dma_proc->read_proc = mic_dma_proc_read_registers_fn; | |
1777 | dma_proc->data = dma_ctx; | |
1778 | } | |
1779 | ||
1780 | } | |
1781 | #endif // LINUX VERSION | |
1782 | ||
1783 | static void | |
1784 | mic_dma_proc_uninit(struct mic_dma_ctx_t *dma_ctx) | |
1785 | { | |
1786 | char name[64]; | |
1787 | ||
1788 | snprintf(name, 63, "%s%d", proc_dma_reg, dma_ctx->device_num); | |
1789 | remove_proc_entry(name, NULL); | |
1790 | snprintf(name, 63, "%s%d", proc_dma_ring, dma_ctx->device_num); | |
1791 | remove_proc_entry(name, NULL); | |
1792 | } |