| 1 | /* |
| 2 | * Copyright 2010-2017 Intel Corporation. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify |
| 5 | * it under the terms of the GNU General Public License, version 2, |
| 6 | * as published by the Free Software Foundation. |
| 7 | * |
| 8 | * This program is distributed in the hope that it will be useful, |
| 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 11 | * General Public License for more details. |
| 12 | * |
| 13 | * Disclaimer: The codes contained in these modules may be specific to |
| 14 | * the Intel Software Development Platform codenamed Knights Ferry, |
| 15 | * and the Intel product codenamed Knights Corner, and are not backward |
| 16 | * compatible with other Intel products. Additionally, Intel will NOT |
| 17 | * support the codes or instruction set in future products. |
| 18 | * |
| 19 | * Intel offers no warranty of any kind regarding the code. This code is |
| 20 | * licensed on an "AS IS" basis and Intel is not obligated to provide |
| 21 | * any support, assistance, installation, training, or other services |
| 22 | * of any kind. Intel is also not obligated to provide any updates, |
| 23 | * enhancements or extensions. Intel specifically disclaims any warranty |
| 24 | * of merchantability, non-infringement, fitness for any particular |
| 25 | * purpose, and any other warranty. |
| 26 | * |
| 27 | * Further, Intel disclaims all liability of any kind, including but |
| 28 | * not limited to liability for infringement of any proprietary rights, |
| 29 | * relating to the use of the code, even if Intel is notified of the |
| 30 | * possibility of such liability. Except as expressly stated in an Intel |
| 31 | * license agreement provided with this code and agreed upon with Intel, |
| 32 | * no license, express or implied, by estoppel or otherwise, to any |
| 33 | * intellectual property rights is granted herein. |
| 34 | */ |
| 35 | |
| 36 | #ifndef COMPL_BUF_RING_H |
| 37 | #define COMPL_BUF_RING_H |
| 38 | #include <linux/errno.h> |
| 39 | #include <linux/hardirq.h> |
| 40 | #include <linux/types.h> |
| 41 | #include <linux/capability.h> |
| 42 | #include <linux/slab.h> |
| 43 | #include <linux/string.h> |
| 44 | #include <linux/gfp.h> |
| 45 | #include <linux/vmalloc.h> |
| 46 | #include <asm/io.h> |
| 47 | #include <linux/kernel.h> |
| 48 | #include <linux/mm_types.h> |
| 49 | #include <linux/jiffies.h> |
| 50 | #include <linux/timer.h> |
| 51 | #include <linux/irqflags.h> |
| 52 | #include <linux/time.h> |
| 53 | #include <linux/spinlock.h> |
| 54 | #include <linux/mutex.h> |
| 55 | #include <linux/semaphore.h> |
| 56 | #include <linux/kthread.h> |
| 57 | #include <linux/sched.h> |
| 58 | #include <linux/delay.h> |
| 59 | #include <linux/wait.h> |
| 60 | #include <asm/bug.h> |
| 61 | #include <linux/pci.h> |
| 62 | #include <linux/device.h> |
| 63 | #include <linux/fs.h> |
| 64 | #include <linux/list.h> |
| 65 | #include <linux/workqueue.h> |
| 66 | #include <linux/interrupt.h> |
| 67 | #include <asm/atomic.h> |
| 68 | #include <linux/netdevice.h> |
| 69 | #include <linux/debugfs.h> |
| 70 | #include "mic_dma_md.h" |
| 71 | #ifndef _MIC_SCIF_ |
| 72 | #include "micscif.h" |
| 73 | #include "micscif_smpt.h" |
| 74 | #endif |
| 75 | #define MAX_POLL_TAIL_READ_RETRIES 20 |
| 76 | |
| 77 | /* |
| 78 | * Assuming read/write to int is atomic |
| 79 | * This can't be used as generic ring because of update_tail() |
| 80 | * One entry is left in the ring to differentiate between ring being empty and |
| 81 | * full |
| 82 | */ |
| 83 | struct compl_buf_ring { |
| 84 | int head; |
| 85 | int tail; |
| 86 | int size; |
| 87 | uint64_t tail_location; |
| 88 | dma_addr_t tail_phys; |
| 89 | }; |
| 90 | |
| 91 | /* |
| 92 | * FIXME: |
| 93 | * Function calls pci_map_single etc, return type needs to indicate |
| 94 | * an error |
| 95 | */ |
| 96 | static __always_inline void init_ring(struct compl_buf_ring *ring, int size, |
| 97 | int device_num) |
| 98 | { |
| 99 | #ifndef _MIC_SCIF_ |
| 100 | struct pci_dev *pdev; |
| 101 | #endif |
| 102 | ring->head = 0; |
| 103 | ring->tail = 0; |
| 104 | ring->size = size; |
| 105 | ring->tail_location = (uint64_t) kmalloc(sizeof(uint64_t), GFP_ATOMIC); |
| 106 | BUG_ON(!ring->tail_location); |
| 107 | *(int*)ring->tail_location = -1; |
| 108 | #ifdef _MIC_SCIF_ |
| 109 | ring->tail_phys = virt_to_phys((void*)ring->tail_location); |
| 110 | #else |
| 111 | micscif_pci_dev(device_num, &pdev); |
| 112 | |
| 113 | ring->tail_phys = mic_map_single(device_num - 1, pdev, (void *)ring->tail_location, |
| 114 | sizeof(uint64_t)); |
| 115 | if (mic_map_error(ring->tail_phys)) |
| 116 | printk(KERN_ERR "mic_map returned error please help\n"); |
| 117 | #endif |
| 118 | } |
| 119 | |
| 120 | static __always_inline void uninit_ring(struct compl_buf_ring *ring, |
| 121 | int device_num) |
| 122 | { |
| 123 | #ifndef _MIC_SCIF_ |
| 124 | struct pci_dev *pdev; |
| 125 | #endif |
| 126 | ring->head = 0; |
| 127 | ring->tail = 0; |
| 128 | ring->size = 0; |
| 129 | #ifndef _MIC_SCIF_ |
| 130 | micscif_pci_dev(device_num, &pdev); |
| 131 | mic_unmap_single(device_num - 1, pdev, ring->tail_phys, sizeof(uint64_t)); |
| 132 | #endif |
| 133 | kfree((void *)ring->tail_location); |
| 134 | } |
| 135 | |
| 136 | static __always_inline int incr_rb_index(int cur_index, int ring_size) |
| 137 | { |
| 138 | return((cur_index + 1) % ring_size); |
| 139 | } |
| 140 | |
| 141 | /* |
| 142 | * Tail location has the index that has been recently processed by dma engine |
| 143 | * But, tail has to point to the index that will be processed next |
| 144 | * So increment the tail |
| 145 | */ |
| 146 | static __always_inline void update_tail(struct compl_buf_ring *ring, int new_tail) |
| 147 | { |
| 148 | ring->tail = new_tail; |
| 149 | } |
| 150 | |
| 151 | static __always_inline int read_tail(struct compl_buf_ring *ring) |
| 152 | { |
| 153 | return incr_rb_index(*(volatile int*)ring->tail_location, ring->size); |
| 154 | } |
| 155 | |
| 156 | /* |
| 157 | * This fn. assumes no one else is updating head |
| 158 | * Returns - avaliable space |
| 159 | * 0 - if no space is available |
| 160 | */ |
| 161 | static __always_inline bool avail_space_in_ring(struct compl_buf_ring *ring) |
| 162 | { |
| 163 | int count = 0, max_num_retries = MAX_POLL_TAIL_READ_RETRIES, num_retries = 0; |
| 164 | int head = ring->head, tail = ring->tail; |
| 165 | retry: |
| 166 | if (head > tail) |
| 167 | count = (tail - 0) + (ring->size - head); |
| 168 | else if (tail > head) |
| 169 | count = tail - head; |
| 170 | else |
| 171 | return ring->size - 1; |
| 172 | |
| 173 | if (1 != count) |
| 174 | return count - 1; |
| 175 | |
| 176 | num_retries++; |
| 177 | if (num_retries == max_num_retries) |
| 178 | return 0; |
| 179 | cpu_relax(); |
| 180 | |
| 181 | ring->tail = read_tail(ring); |
| 182 | tail = ring->tail; |
| 183 | |
| 184 | goto retry; |
| 185 | } |
| 186 | |
| 187 | /* |
| 188 | * Used for polling |
| 189 | */ |
| 190 | static __always_inline bool is_entry_processed(struct compl_buf_ring *ring, int index) |
| 191 | { |
| 192 | int head = ring->head, tail = ring->tail; |
| 193 | if (head < tail) { |
| 194 | if (index >= head && index < tail) |
| 195 | return 1; |
| 196 | } else { |
| 197 | if (index >= head || index < tail) |
| 198 | return 1; |
| 199 | } |
| 200 | return 0; |
| 201 | } |
| 202 | |
| 203 | static __always_inline void incr_head(struct compl_buf_ring *ring) |
| 204 | { |
| 205 | ring->head = incr_rb_index(ring->head, ring->size); |
| 206 | } |
| 207 | |
| 208 | /* |
| 209 | * This function is not reentrant |
| 210 | * It is expected that the user of this func, will call incr_head() if allocated |
| 211 | * buffer is used |
| 212 | */ |
| 213 | static __always_inline int allocate_buffer(struct compl_buf_ring *ring) |
| 214 | { |
| 215 | if (avail_space_in_ring(ring)) |
| 216 | return ring->head; |
| 217 | else |
| 218 | return -1; |
| 219 | } |
| 220 | #endif |