| 1 | /* |
| 2 | * Copyright 2010-2017 Intel Corporation. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify |
| 5 | * it under the terms of the GNU General Public License, version 2, |
| 6 | * as published by the Free Software Foundation. |
| 7 | * |
| 8 | * This program is distributed in the hope that it will be useful, |
| 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 11 | * General Public License for more details. |
| 12 | * |
| 13 | * Disclaimer: The codes contained in these modules may be specific to |
| 14 | * the Intel Software Development Platform codenamed Knights Ferry, |
| 15 | * and the Intel product codenamed Knights Corner, and are not backward |
| 16 | * compatible with other Intel products. Additionally, Intel will NOT |
| 17 | * support the codes or instruction set in future products. |
| 18 | * |
| 19 | * Intel offers no warranty of any kind regarding the code. This code is |
| 20 | * licensed on an "AS IS" basis and Intel is not obligated to provide |
| 21 | * any support, assistance, installation, training, or other services |
| 22 | * of any kind. Intel is also not obligated to provide any updates, |
| 23 | * enhancements or extensions. Intel specifically disclaims any warranty |
| 24 | * of merchantability, non-infringement, fitness for any particular |
| 25 | * purpose, and any other warranty. |
| 26 | * |
| 27 | * Further, Intel disclaims all liability of any kind, including but |
| 28 | * not limited to liability for infringement of any proprietary rights, |
| 29 | * relating to the use of the code, even if Intel is notified of the |
| 30 | * possibility of such liability. Except as expressly stated in an Intel |
| 31 | * license agreement provided with this code and agreed upon with Intel, |
| 32 | * no license, express or implied, by estoppel or otherwise, to any |
| 33 | * intellectual property rights is granted herein. |
| 34 | */ |
| 35 | |
| 36 | #include "mic_common.h" |
| 37 | #include "mic/micscif_smpt.h" |
| 38 | #include "mic/micscif_nodeqp.h" |
| 39 | #include "mic/micscif_intr.h" |
| 40 | #include "mic/micscif_nm.h" |
| 41 | #include "micint.h" |
| 42 | |
| 43 | struct micscif_info ms_info; |
| 44 | struct micscif_dev scif_dev[MAX_BOARD_SUPPORTED + 1]; |
| 45 | |
| 46 | bool mic_watchdog_enable = 1; |
| 47 | bool mic_watchdog_auto_reboot = 1; |
| 48 | bool mic_crash_dump_enabled = 1; |
| 49 | |
| 50 | int |
| 51 | micscif_init(void) |
| 52 | { |
| 53 | int err; |
| 54 | ms_info.mi_nodeid = 0; // Host is node 0 |
| 55 | ms_info.mi_maxid = 0; // Host is at start the max card ID |
| 56 | ms_info.mi_total = 1; // Host will know about this many MIC cards |
| 57 | ms_info.mi_mask = 1; // first bit in the mask is the host node |
| 58 | |
| 59 | mutex_init (&ms_info.mi_conflock); |
| 60 | spin_lock_init(&ms_info.mi_eplock); |
| 61 | spin_lock_init(&ms_info.mi_connlock); |
| 62 | spin_lock_init(&ms_info.mi_rmalock); |
| 63 | mutex_init (&ms_info.mi_fencelock); |
| 64 | mutex_init (&ms_info.mi_event_cblock); |
| 65 | spin_lock_init(&ms_info.mi_nb_connect_lock); |
| 66 | INIT_LIST_HEAD(&ms_info.mi_uaccept); |
| 67 | INIT_LIST_HEAD(&ms_info.mi_listen); |
| 68 | INIT_LIST_HEAD(&ms_info.mi_zombie); |
| 69 | INIT_LIST_HEAD(&ms_info.mi_connected); |
| 70 | INIT_LIST_HEAD(&ms_info.mi_disconnected); |
| 71 | INIT_LIST_HEAD(&ms_info.mi_rma); |
| 72 | INIT_LIST_HEAD(&ms_info.mi_rma_tc); |
| 73 | #ifdef CONFIG_MMU_NOTIFIER |
| 74 | INIT_LIST_HEAD(&ms_info.mi_mmu_notif_cleanup); |
| 75 | #endif |
| 76 | INIT_LIST_HEAD(&ms_info.mi_fence); |
| 77 | INIT_LIST_HEAD(&ms_info.mi_event_cb); |
| 78 | INIT_LIST_HEAD(&ms_info.mi_nb_connect_list); |
| 79 | ms_info.mi_watchdog_to = DEFAULT_WATCHDOG_TO; |
| 80 | #ifdef MIC_IS_EMULATION |
| 81 | ms_info.mi_watchdog_enabled = 0; |
| 82 | ms_info.mi_watchdog_auto_reboot = 0; |
| 83 | #else |
| 84 | ms_info.mi_watchdog_enabled = mic_watchdog_enable; |
| 85 | ms_info.mi_watchdog_auto_reboot = mic_watchdog_auto_reboot; |
| 86 | #endif |
| 87 | #ifdef RMA_DEBUG |
| 88 | ms_info.rma_unaligned_cpu_cnt = (atomic_long_t) ATOMIC_LONG_INIT(0); |
| 89 | ms_info.rma_alloc_cnt = (atomic_long_t) ATOMIC_LONG_INIT(0); |
| 90 | ms_info.rma_pin_cnt = (atomic_long_t) ATOMIC_LONG_INIT(0); |
| 91 | #ifdef CONFIG_MMU_NOTIFIER |
| 92 | ms_info.mmu_notif_cnt = (atomic_long_t) ATOMIC_LONG_INIT(0); |
| 93 | #endif |
| 94 | #endif |
| 95 | ms_info.mi_misc_wq = __mic_create_singlethread_workqueue("SCIF_MISC"); |
| 96 | if (!ms_info.mi_misc_wq) { |
| 97 | err = -ENOMEM; |
| 98 | goto wq_error; |
| 99 | } |
| 100 | INIT_WORK(&ms_info.mi_misc_work, micscif_misc_handler); |
| 101 | #ifdef CONFIG_MMU_NOTIFIER |
| 102 | ms_info.mi_mmu_notif_wq = create_singlethread_workqueue("SCIF_MMU"); |
| 103 | if (!ms_info.mi_mmu_notif_wq) { |
| 104 | err = -ENOMEM; |
| 105 | goto wq_error; |
| 106 | } |
| 107 | INIT_WORK(&ms_info.mi_mmu_notif_work, micscif_mmu_notif_handler); |
| 108 | #endif |
| 109 | ms_info.mi_conn_wq = __mic_create_singlethread_workqueue("SCIF_NB_CONN"); |
| 110 | if (!ms_info.mi_conn_wq) { |
| 111 | err = -ENOMEM; |
| 112 | goto wq_error; |
| 113 | } |
| 114 | INIT_WORK(&ms_info.mi_conn_work, micscif_conn_handler); |
| 115 | |
| 116 | //pr_debug("micscif_create(%d) \n", num_bds); |
| 117 | |
| 118 | // Setup information for self aka loopback. |
| 119 | scif_dev[SCIF_HOST_NODE].sd_node = SCIF_HOST_NODE; |
| 120 | micscif_setup_loopback_qp(&scif_dev[SCIF_HOST_NODE]); |
| 121 | scif_dev[SCIF_HOST_NODE].sd_state = SCIFDEV_RUNNING; |
| 122 | scif_dev[SCIF_HOST_NODE].scif_ref_cnt = |
| 123 | (atomic_long_t) ATOMIC_LONG_INIT(0); |
| 124 | scif_dev[SCIF_HOST_NODE].scif_map_ref_cnt = 0; |
| 125 | init_waitqueue_head(&scif_dev[SCIF_HOST_NODE].sd_wq); |
| 126 | init_waitqueue_head(&scif_dev[SCIF_HOST_NODE].sd_mmap_wq); |
| 127 | mutex_init (&scif_dev[SCIF_HOST_NODE].sd_lock); |
| 128 | ms_info.mi_rma_tc_limit = SCIF_RMA_TEMP_CACHE_LIMIT; |
| 129 | ms_info.en_msg_log = 0; |
| 130 | scif_proc_init(); |
| 131 | return 0; |
| 132 | wq_error: |
| 133 | if (ms_info.mi_misc_wq) |
| 134 | destroy_workqueue(ms_info.mi_misc_wq); |
| 135 | #ifdef CONFIG_MMU_NOTIFIER |
| 136 | if (ms_info.mi_mmu_notif_wq) |
| 137 | destroy_workqueue(ms_info.mi_mmu_notif_wq); |
| 138 | #endif |
| 139 | if (ms_info.mi_conn_wq) |
| 140 | destroy_workqueue(ms_info.mi_conn_wq); |
| 141 | return err; |
| 142 | } |
| 143 | |
| 144 | void |
| 145 | micscif_destroy(void) |
| 146 | { |
| 147 | struct list_head *pos, *unused; |
| 148 | struct scif_callback *temp; |
| 149 | #ifdef CONFIG_MMU_NOTIFIER |
| 150 | destroy_workqueue(ms_info.mi_mmu_notif_wq); |
| 151 | #endif |
| 152 | destroy_workqueue(ms_info.mi_misc_wq); |
| 153 | destroy_workqueue(ms_info.mi_conn_wq); |
| 154 | micscif_destroy_loopback_qp(&scif_dev[SCIF_HOST_NODE]); |
| 155 | scif_proc_cleanup(); |
| 156 | mic_debug_uninit(); |
| 157 | list_for_each_safe(pos, unused, &ms_info.mi_event_cb) { |
| 158 | temp = list_entry(pos, struct scif_callback, list_member); |
| 159 | list_del(pos); |
| 160 | kfree(temp); |
| 161 | } |
| 162 | mutex_destroy(&ms_info.mi_event_cblock); |
| 163 | } |
| 164 | |
| 165 | int |
| 166 | micscif_host_doorbell_intr_handler(mic_ctx_t *mic_ctx, int doorbell) |
| 167 | { |
| 168 | struct micscif_dev *dev = &scif_dev[mic_ctx->bi_id + 1]; |
| 169 | |
| 170 | queue_work(dev->sd_intr_wq, &dev->sd_intr_bh); |
| 171 | return 0; |
| 172 | } |
| 173 | |
| 174 | int micscif_setup_host_qp(mic_ctx_t *mic_ctx, struct micscif_dev *scifdev); |
| 175 | |
| 176 | void |
| 177 | micscif_probe(mic_ctx_t *mic_ctx) |
| 178 | { |
| 179 | struct micscif_dev *scifdev = &scif_dev[mic_ctx->bi_id + 1]; |
| 180 | |
| 181 | // The host needs to keep track of scif_dev interfaces for all boards in |
| 182 | // the system. Host is node zero for MIC board 0 is SCIF node 1, etc. |
| 183 | // This will need to become more dynamic if hot plug is supported |
| 184 | |
| 185 | scifdev->sd_node = mic_ctx->bi_id + 1; |
| 186 | scifdev->sd_state = SCIFDEV_STOPPED; |
| 187 | scifdev->mm_sbox = mic_ctx->mmio.va + HOST_SBOX_BASE_ADDRESS; |
| 188 | |
| 189 | /* This workqueue thread will handle all card->host interrupt processing. */ |
| 190 | micscif_setup_interrupts(scifdev); |
| 191 | |
| 192 | init_waitqueue_head(&scifdev->sd_mmap_wq); |
| 193 | init_waitqueue_head(&scifdev->sd_wq); |
| 194 | mutex_init (&scifdev->sd_lock); |
| 195 | INIT_LIST_HEAD(&scifdev->sd_p2p); |
| 196 | |
| 197 | init_waitqueue_head(&scifdev->sd_watchdog_wq); |
| 198 | snprintf(scifdev->sd_ln_wqname, sizeof(scifdev->sd_intr_wqname), |
| 199 | "SCIF LOSTNODE %d", scifdev->sd_node); |
| 200 | if (!(scifdev->sd_ln_wq = |
| 201 | __mic_create_singlethread_workqueue(scifdev->sd_ln_wqname))) |
| 202 | printk(KERN_ERR "%s %d wq creation failed\n", __func__, __LINE__); |
| 203 | INIT_DELAYED_WORK(&scifdev->sd_watchdog_work, micscif_watchdog_handler); |
| 204 | /* |
| 205 | * Register function for doorbell 0 which will |
| 206 | * basically kick off the workqueue. |
| 207 | */ |
| 208 | mic_reg_irqhandler(mic_ctx, 0, "SCIF DoorBell 0", |
| 209 | micscif_host_doorbell_intr_handler); |
| 210 | } |
| 211 | |
| 212 | void |
| 213 | micscif_start(mic_ctx_t *mic_ctx) |
| 214 | { |
| 215 | struct micscif_dev *scifdev = &scif_dev[mic_ctx->bi_id + 1]; |
| 216 | |
| 217 | scifdev->scif_ref_cnt = (atomic_long_t) ATOMIC_LONG_INIT(0); |
| 218 | scifdev->scif_map_ref_cnt = 0; |
| 219 | |
| 220 | scifdev->sd_state = SCIFDEV_INIT; |
| 221 | |
| 222 | |
| 223 | /* Sets up bd_bs and the host side of the queuepair */ |
| 224 | pr_debug("micscif_probe: host setting up qp \n"); |
| 225 | micscif_setup_host_qp(mic_ctx, scifdev); |
| 226 | } |
| 227 | |
| 228 | void micscif_removehost_respose(struct micscif_dev *scifdev, struct nodemsg *msg); |
| 229 | |
| 230 | void |
| 231 | micscif_stop(mic_ctx_t *mic_ctx) |
| 232 | { |
| 233 | struct micscif_dev *scifdev = &scif_dev[mic_ctx->bi_id + 1]; |
| 234 | |
| 235 | if (scifdev->sd_state == SCIFDEV_STOPPED || scifdev->sd_state == SCIFDEV_INIT) |
| 236 | return; |
| 237 | |
| 238 | micscif_disconnect_node(scifdev->sd_node, NULL, DISCONN_TYPE_LOST_NODE); |
| 239 | } |
| 240 | |
| 241 | void |
| 242 | micscif_remove(mic_ctx_t *mic_ctx) |
| 243 | { |
| 244 | struct micscif_dev *scifdev = &scif_dev[mic_ctx->bi_id + 1]; |
| 245 | struct micscif_qp *qp = &scifdev->qpairs[0]; |
| 246 | |
| 247 | destroy_workqueue(scifdev->sd_intr_wq); |
| 248 | scifdev->sd_intr_wq = 0; |
| 249 | cancel_delayed_work_sync(&scifdev->sd_watchdog_work); |
| 250 | if (scifdev->sd_ln_wq){ |
| 251 | destroy_workqueue(scifdev->sd_ln_wq); |
| 252 | scifdev->sd_ln_wq = 0; |
| 253 | } |
| 254 | mic_unreg_irqhandler(mic_ctx, 0x0, "SCIF DoorBell 0"); |
| 255 | |
| 256 | if (qp) { |
| 257 | mic_ctx_unmap_single(mic_ctx, qp->local_buf, qp->inbound_q.size); |
| 258 | mic_ctx_unmap_single(mic_ctx, qp->local_qp, sizeof(struct micscif_qp)); |
| 259 | kfree((void*)(qp->inbound_q.rb_base)); |
| 260 | } |
| 261 | |
| 262 | if (scifdev->qpairs) { |
| 263 | kfree(scifdev->qpairs); |
| 264 | scifdev->qpairs = NULL; |
| 265 | } |
| 266 | } |
| 267 | |
| 268 | int |
| 269 | scif_get_node_status(int node_id) |
| 270 | { |
| 271 | struct micscif_dev *scifdev = &scif_dev[node_id]; |
| 272 | |
| 273 | return scifdev->sd_state; |
| 274 | } |
| 275 | |
| 276 | struct scatterlist * |
| 277 | micscif_p2p_mapsg(void *va, int page_size, int page_cnt) |
| 278 | { |
| 279 | struct scatterlist *sg; |
| 280 | struct page *page; |
| 281 | int i; |
| 282 | |
| 283 | if ((sg = kcalloc(page_cnt, sizeof(struct scatterlist), GFP_KERNEL)) == NULL) { |
| 284 | return NULL; |
| 285 | } |
| 286 | |
| 287 | sg_init_table(sg, page_cnt); |
| 288 | |
| 289 | for (i = 0; i < page_cnt; i++) { |
| 290 | #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0)) |
| 291 | phys_addr_t phys; |
| 292 | phys = slow_virt_to_phys(va); |
| 293 | |
| 294 | if ((page = pfn_to_page(phys >> PAGE_SHIFT)) == NULL) |
| 295 | goto p2p_sg_err; |
| 296 | #else |
| 297 | if ((page = vmalloc_to_page(va)) == NULL) |
| 298 | goto p2p_sg_err; |
| 299 | #endif |
| 300 | sg_set_page(&sg[i], page, page_size, 0); |
| 301 | va += page_size; |
| 302 | } |
| 303 | |
| 304 | return sg; |
| 305 | |
| 306 | p2p_sg_err: |
| 307 | kfree(sg); |
| 308 | return NULL; |
| 309 | } |
| 310 | |
| 311 | void |
| 312 | micscif_p2p_freesg(struct scatterlist *sg) |
| 313 | { |
| 314 | kfree(sg); |
| 315 | } |