Commit | Line | Data |
---|---|---|
800f879a AT |
1 | |
2 | /* | |
3 | * Copyright 2010-2017 Intel Corporation. | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License, version 2, | |
7 | * as published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
12 | * General Public License for more details. | |
13 | * | |
14 | * Disclaimer: The codes contained in these modules may be specific to | |
15 | * the Intel Software Development Platform codenamed Knights Ferry, | |
16 | * and the Intel product codenamed Knights Corner, and are not backward | |
17 | * compatible with other Intel products. Additionally, Intel will NOT | |
18 | * support the codes or instruction set in future products. | |
19 | * | |
20 | * Intel offers no warranty of any kind regarding the code. This code is | |
21 | * licensed on an "AS IS" basis and Intel is not obligated to provide | |
22 | * any support, assistance, installation, training, or other services | |
23 | * of any kind. Intel is also not obligated to provide any updates, | |
24 | * enhancements or extensions. Intel specifically disclaims any warranty | |
25 | * of merchantability, non-infringement, fitness for any particular | |
26 | * purpose, and any other warranty. | |
27 | * | |
28 | * Further, Intel disclaims all liability of any kind, including but | |
29 | * not limited to liability for infringement of any proprietary rights, | |
30 | * relating to the use of the code, even if Intel is notified of the | |
31 | * possibility of such liability. Except as expressly stated in an Intel | |
32 | * license agreement provided with this code and agreed upon with Intel, | |
33 | * no license, express or implied, by estoppel or otherwise, to any | |
34 | * intellectual property rights is granted herein. | |
35 | */ | |
36 | ||
37 | #include <linux/ip.h> | |
38 | #include <linux/tcp.h> | |
39 | #include <linux/list.h> | |
40 | #include <linux/circ_buf.h> | |
41 | #include <linux/reboot.h> | |
42 | #include "mic_common.h" | |
43 | #include "mic/micveth_dma.h" | |
44 | #include "mic/mic_macaddr.h" | |
45 | ||
46 | /* TODO: Clean up shutdown, let DMA's drain */ | |
47 | ||
48 | #ifndef HOST | |
49 | #define SBOX_SDBIC0_DBREQ_BIT 0x80000000 | |
50 | #define SBOX_MMIO_LENGTH (64 * 1024) | |
51 | #endif | |
52 | #define STOP_WAIT_TIMEOUT (4 * HZ) | |
53 | ||
54 | #ifndef HOST | |
55 | static mic_ctx_t mic_ctx_g; | |
56 | #endif | |
57 | ||
58 | struct micvnet micvnet; | |
59 | ||
60 | ||
61 | static void micvnet_send_intr(struct micvnet_info *vnet_info); | |
62 | static int micvnet_init_msg_rings(struct micvnet_info *vnet_info); | |
63 | static int micvnet_init_rx_skb_send_msg(struct micvnet_info *vnet_info); | |
64 | static void micvnet_send_add_dma_buffer_messages(struct micvnet_info *vnet_info); | |
65 | static void micvnet_stop_ws(struct work_struct *work); | |
66 | static void micvnet_start_ws(struct work_struct *work); | |
67 | int get_sbox_irq(int index); | |
68 | ||
69 | static __always_inline mic_ctx_t * | |
70 | vnet_to_ctx(struct micvnet_info *vnet_info) | |
71 | { | |
72 | return vnet_info->mic_ctx; | |
73 | } | |
74 | ||
75 | static __always_inline void | |
76 | micvnet_wake_queue(struct micvnet_info *vnet_info) | |
77 | { | |
78 | if (atomic_read(&vnet_info->vi_state) == MICVNET_STATE_LINKUP) | |
79 | netif_wake_queue(vnet_info->vi_netdev); | |
80 | } | |
81 | ||
82 | static __always_inline void | |
83 | micvnet_dec_cnt_tx_pending(struct micvnet_info *vnet_info) | |
84 | { | |
85 | if (atomic_dec_and_test(&vnet_info->cnt_tx_pending) && | |
86 | (atomic_read(&vnet_info->vi_state) == MICVNET_STATE_LINK_DOWN)) | |
87 | wake_up_interruptible(&vnet_info->stop_waitq); | |
88 | } | |
89 | ||
90 | ||
91 | /*********************************************************** | |
92 | Pre-allocated "list" of objects which are allocated and deallocated in FIFO | |
93 | sequence. Allows reservation of memory at init time to prevent mem allocation | |
94 | failures at run time. */ | |
95 | static int | |
96 | list_obj_list_init(int num_obj, size_t obj_size, struct obj_list *list) | |
97 | { | |
98 | list->size = num_obj + 1; | |
99 | list->obj_size = obj_size; | |
100 | list->head = list->tail = 0; | |
101 | ||
102 | if (!(list->buf = kmalloc(list->size * list->obj_size, GFP_KERNEL))) { | |
103 | printk(KERN_ERR "%s: list alloc failed\n", __func__); | |
104 | return -ENOMEM; | |
105 | } | |
106 | return 0; | |
107 | } | |
108 | ||
109 | static void | |
110 | list_obj_list_deinit(struct obj_list *list) | |
111 | { | |
112 | if (list->buf) { | |
113 | kfree(list->buf); | |
114 | list->buf = NULL; | |
115 | } | |
116 | } | |
117 | ||
118 | static void * | |
119 | list_obj_alloc(struct obj_list *list) | |
120 | { | |
121 | char *obj; | |
122 | ||
123 | /* Remove bug_on() here to handle VNET OOO messages. In OOO conditions | |
124 | * requests to allocate more objects than list->size are possible. */ | |
125 | if (((list->head + 1) % list->size) == list->tail) { | |
126 | printk(KERN_ERR "%s: BUG: no free objects in obj list\n", __func__); | |
127 | return NULL; | |
128 | } | |
129 | ||
130 | obj = list->buf + list->head * list->obj_size; | |
131 | wmb(); | |
132 | list->head = (list->head + 1) % list->size; | |
133 | ||
134 | return obj; | |
135 | } | |
136 | ||
137 | void | |
138 | list_obj_free(struct obj_list *list) | |
139 | { | |
140 | /* Remove bug_on() here to handle VNET OOO messages */ | |
141 | if (list->tail == list->head) { | |
142 | printk(KERN_ERR "%s: BUG: free too many list objects\n", __func__); | |
143 | return; | |
144 | } | |
145 | ||
146 | list->tail = (list->tail + 1) % list->size; | |
147 | } | |
148 | ||
149 | /*********************************************************** | |
150 | * Vnet message functions | |
151 | */ | |
152 | #ifdef HOST | |
153 | static void | |
154 | micvnet_msg_rb_init(struct micvnet_msg_rb *rb) | |
155 | { | |
156 | rb->head = rb->tail = 0; | |
157 | rb->size = MICVNET_MSG_RB_SIZE; | |
158 | rb->prev_head = rb->prev_tail = rb->size - 1; | |
159 | } | |
160 | ||
161 | static void | |
162 | micvnet_reset_msg_rings(struct micvnet_info *vnet_info) | |
163 | { | |
164 | micvnet_msg_rb_init(vnet_info->vi_qp.tx); | |
165 | micvnet_msg_rb_init(vnet_info->vi_qp.rx); | |
166 | } | |
167 | #endif | |
168 | ||
169 | static void | |
170 | micvnet_msg_rb_write_msg(struct micvnet_info *vnet_info, struct micvnet_msg *msg) | |
171 | { | |
172 | struct micvnet_msg_rb *rb = vnet_info->vi_qp.tx; | |
173 | ||
174 | /* The condition below should never occur under normal conditions | |
175 | because the VNET message ring buffer size is at least 1 greater than | |
176 | the maximum total number of outstanding messages possible in the | |
177 | system. However, all bets are off if VNET OOO messages are | |
178 | seen. Therefore remove the previous bug_on() here and busy wait. */ | |
179 | while (((rb->head + 1) % rb->size) == rb->tail) | |
180 | cpu_relax(); | |
181 | ||
182 | if (!(rb->head == (rb->prev_head + 1) % rb->size)) | |
183 | printk(KERN_ERR "BUG: head not equal to prev_head + 1:\n \ | |
184 | head %d prev_head %d\n", rb->head, rb->prev_head); | |
185 | ||
186 | smp_mb(); | |
187 | #ifdef HOST | |
188 | rb->buf[rb->head] = *msg; | |
189 | #else | |
190 | memcpy_toio(&rb->buf[rb->head], msg, sizeof(*msg)); | |
191 | #endif | |
192 | smp_mb(); | |
193 | serializing_request(&rb->buf[rb->head]); | |
194 | ||
195 | rb->prev_head = rb->head; | |
196 | rb->head = (rb->head + 1) % rb->size; | |
197 | #ifndef HOST | |
198 | rb->head = rb->head; | |
199 | #endif | |
200 | smp_mb(); | |
201 | serializing_request(&rb->head); | |
202 | } | |
203 | ||
204 | static int | |
205 | micvnet_msg_rb_read_msg(struct micvnet_info *vnet_info, struct micvnet_msg *msg) | |
206 | { | |
207 | struct micvnet_msg_rb *rb = vnet_info->vi_qp.rx; | |
208 | ||
209 | if (rb->tail == rb->head) | |
210 | return 1; | |
211 | ||
212 | if (!(rb->tail == (rb->prev_tail + 1) % rb->size)) | |
213 | printk(KERN_ERR "BUG: tail not equal to prev_tail + 1:\n \ | |
214 | tail %d prev_tail %d\n", rb->tail, rb->prev_tail); | |
215 | ||
216 | smp_mb(); | |
217 | #ifdef HOST | |
218 | *msg = rb->buf[rb->tail]; | |
219 | #else | |
220 | memcpy_fromio(msg, &rb->buf[rb->tail], sizeof(*msg)); | |
221 | #endif | |
222 | smp_mb(); | |
223 | serializing_request(&rb->buf[rb->tail]); | |
224 | ||
225 | rb->prev_tail = rb->tail; | |
226 | rb->tail = (rb->tail + 1) % rb->size; | |
227 | #ifndef HOST | |
228 | rb->tail = rb->tail; | |
229 | #endif | |
230 | smp_mb(); | |
231 | serializing_request(&rb->tail); | |
232 | ||
233 | return 0; | |
234 | } | |
235 | ||
236 | void | |
237 | micvnet_msg_send_msg(struct micvnet_info *vnet_info, struct micvnet_msg *msg) | |
238 | { | |
239 | micvnet_msg_rb_write_msg(vnet_info, msg); | |
240 | #ifdef HOST | |
241 | if (micpm_get_reference(vnet_to_ctx(vnet_info), true)) | |
242 | return; | |
243 | #endif | |
244 | micvnet_send_intr(vnet_info); | |
245 | #ifdef HOST | |
246 | micpm_put_reference(vnet_to_ctx(vnet_info)); | |
247 | #endif | |
248 | } | |
249 | ||
250 | static void | |
251 | micvnet_msg_send_add_dma_buffer_msg(struct micvnet_info *vnet_info, | |
252 | struct rx_node *rnode) | |
253 | { | |
254 | struct micvnet_msg msg; | |
255 | struct micvnet_msg_add_dma_buffer | |
256 | *body = &msg.body.micvnet_msg_add_dma_buffer; | |
257 | ||
258 | msg.msg_id = MICVNET_MSG_ADD_DMA_BUFFER; | |
259 | body->buf_phys = rnode->phys; | |
260 | body->buf_size = rnode->size; | |
261 | micvnet_msg_send_msg(vnet_info, &msg); | |
262 | } | |
263 | ||
264 | static void | |
265 | micvnet_msg_recv_add_dma_buffer(struct micvnet_info *vnet_info, | |
266 | struct micvnet_msg_add_dma_buffer *msg) | |
267 | { | |
268 | struct dma_node *dnode; | |
269 | ||
270 | /* Remove bug_on() here to handle VNET OOO messages */ | |
271 | if (!(dnode = list_obj_alloc(&vnet_info->dnode_list))) | |
272 | return; | |
273 | ||
274 | dnode->phys = msg->buf_phys; | |
275 | dnode->size = msg->buf_size; | |
276 | ||
277 | spin_lock(&vnet_info->vi_rxlock); | |
278 | list_add_tail(&dnode->list, &vnet_info->vi_dma_buf); | |
279 | spin_unlock(&vnet_info->vi_rxlock); | |
280 | ||
281 | atomic_inc(&vnet_info->cnt_dma_buf_avail); | |
282 | micvnet_wake_queue(vnet_info); | |
283 | } | |
284 | ||
285 | static void | |
286 | micvnet_msg_send_dma_complete_msg(struct micvnet_info *vnet_info, | |
287 | struct sched_node *snode) | |
288 | { | |
289 | struct micvnet_msg msg; | |
290 | struct micvnet_msg_dma_complete | |
291 | *body = &msg.body.micvnet_msg_dma_complete; | |
292 | ||
293 | msg.msg_id = MICVNET_MSG_DMA_COMPLETE; | |
294 | body->dst_phys = snode->dst_phys; | |
295 | body->size = snode->skb->len; | |
296 | body->dma_offset = snode->dma_offset; | |
297 | micvnet_msg_send_msg(vnet_info, &msg); | |
298 | } | |
299 | ||
300 | /* Handle an unexpected out-of-order message */ | |
301 | static int | |
302 | micvnet_msg_handle_ooo_msg(struct micvnet_info *vnet_info, | |
303 | struct micvnet_msg_dma_complete *msg) | |
304 | { | |
305 | struct micvnet_msg_rb *rb = vnet_info->vi_qp.rx; | |
306 | struct rx_node *rnode; | |
307 | struct list_head *pos, *tmpl; | |
308 | bool found = false; | |
309 | ||
310 | rnode = list_entry((&vnet_info->vi_rx_skb)->next, struct rx_node, list); | |
311 | ||
312 | /* Normal operation */ | |
313 | if (rnode->phys == msg->dst_phys | |
314 | && msg->size <= (rnode->size - 3 * DMA_ALIGNMENT) | |
315 | && msg->dma_offset < 2 * DMA_ALIGNMENT) | |
316 | return 0; | |
317 | ||
318 | /* Flag that weird stuff's going on */ | |
319 | printk(KERN_ERR "BUG: Unexpected vnet dma_complete message parameters:\n \ | |
320 | rnode->phys %p, msg->dst_phys %p\n \ | |
321 | rnode->size %lld, msg->size %lld, msg->dma_offset %lld\n \ | |
322 | rx rb head %d tail %d size %d\n", | |
323 | (char *) rnode->phys, (char *) msg->dst_phys, | |
324 | rnode->size, msg->size, msg->dma_offset, | |
325 | rb->head, rb->tail, rb->size); | |
326 | ||
327 | /* if message is received in order but with incorrect parameters | |
328 | (size/dma_offset), drop it, but re-add the rnode at the back of the | |
329 | rx_skb list, as well as at tx, similar to what is done below for ooo | |
330 | case. */ | |
331 | if (rnode->phys == msg->dst_phys) { | |
332 | list_del(&rnode->list); | |
333 | list_add_tail(&rnode->list, &vnet_info->vi_rx_skb); | |
334 | micvnet_msg_send_add_dma_buffer_msg(vnet_info, rnode); | |
335 | vnet_info->vi_netdev->stats.rx_dropped++; | |
336 | return 1; | |
337 | } | |
338 | ||
339 | /* Start of OOO message processing. First check if the message has | |
340 | * really been received OOO. If it is completely unknown to us we just | |
341 | * drop it and go on. */ | |
342 | list_for_each(pos, &vnet_info->vi_rx_skb) { | |
343 | rnode = list_entry(pos, struct rx_node, list); | |
344 | if (rnode->phys == msg->dst_phys) { | |
345 | found = true; | |
346 | break; | |
347 | } | |
348 | } | |
349 | ||
350 | if (!found) { | |
351 | vnet_info->vi_netdev->stats.rx_dropped++; | |
352 | return 1; | |
353 | } | |
354 | ||
355 | vnet_info->vi_netdev->stats.rx_errors++; | |
356 | ||
357 | /* Skip all the rnode's till we find the one we are looking for. Rather | |
358 | * than free rnode skb's and reallocate them, and therby risk allocation | |
359 | * failures, we simply delete the rnode's from their current position on | |
360 | * the rnode list and re-add them at back of the list, as well as add | |
361 | * them back at tx. */ | |
362 | list_for_each_safe(pos, tmpl, &vnet_info->vi_rx_skb) { | |
363 | rnode = list_entry(pos, struct rx_node, list); | |
364 | if (rnode->phys == msg->dst_phys) | |
365 | break; | |
366 | ||
367 | list_del(&rnode->list); | |
368 | list_add_tail(&rnode->list, &vnet_info->vi_rx_skb); | |
369 | micvnet_msg_send_add_dma_buffer_msg(vnet_info, rnode); | |
370 | } | |
371 | ||
372 | return 0; | |
373 | } | |
374 | ||
375 | static void | |
376 | micvnet_msg_recv_dma_complete(struct micvnet_info *vnet_info, | |
377 | struct micvnet_msg_dma_complete *msg) | |
378 | { | |
379 | struct rx_node *rnode; | |
380 | struct sk_buff *skb; | |
381 | ||
382 | vnet_info->vi_netdev->stats.rx_packets++; | |
383 | ||
384 | if (micvnet_msg_handle_ooo_msg(vnet_info, msg)) | |
385 | return; | |
386 | ||
387 | rnode = list_entry((&vnet_info->vi_rx_skb)->next, struct rx_node, list); | |
388 | /* Our OOO message handling guarantees that rnode->phys == msg->dst_phys */ | |
389 | ||
390 | vnet_info->vi_netdev->stats.rx_bytes += msg->size; | |
391 | list_del(&rnode->list); | |
392 | ||
393 | spin_lock_bh(&vnet_info->vi_txlock); | |
394 | if (atomic_read(&vnet_info->vi_state) != MICVNET_STATE_LINKUP) { | |
395 | spin_unlock_bh(&vnet_info->vi_txlock); | |
396 | goto skip_adding_new_buffers; | |
397 | } | |
398 | atomic_inc(&vnet_info->cnt_tx_pending); | |
399 | spin_unlock_bh(&vnet_info->vi_txlock); | |
400 | ||
401 | /* OOM handling: check if a new SKB can be allocated. If not, we will re-add the | |
402 | old SKB to TX and not give it to the network stack, i.e. drop it */ | |
403 | if (micvnet_init_rx_skb_send_msg(vnet_info)) { | |
404 | list_add_tail(&rnode->list, &vnet_info->vi_rx_skb); | |
405 | micvnet_msg_send_add_dma_buffer_msg(vnet_info, rnode); | |
406 | micvnet_dec_cnt_tx_pending(vnet_info); | |
407 | vnet_info->vi_netdev->stats.rx_dropped++; | |
408 | return; | |
409 | } | |
410 | micvnet_dec_cnt_tx_pending(vnet_info); | |
411 | ||
412 | skip_adding_new_buffers: | |
413 | skb = rnode->skb; | |
414 | skb_reserve(skb, msg->dma_offset); | |
415 | skb_put(skb, msg->size); | |
416 | skb->dev = vnet_info->vi_netdev; | |
417 | skb->protocol = eth_type_trans(skb, skb->dev); | |
418 | skb->ip_summed = CHECKSUM_NONE; | |
419 | ||
420 | local_bh_disable(); | |
421 | netif_receive_skb(skb); | |
422 | local_bh_enable(); | |
423 | ||
424 | #ifdef HOST | |
425 | mic_ctx_unmap_single(vnet_to_ctx(vnet_info), rnode->phys, rnode->size); | |
426 | #endif | |
427 | kfree(rnode); | |
428 | } | |
429 | ||
430 | static void | |
431 | micvnet_msg_send_link_down_msg(struct work_struct *work) | |
432 | { | |
433 | struct micvnet_info *vnet_info | |
434 | = container_of(work, struct micvnet_info, vi_ws_link_down); | |
435 | struct micvnet_msg msg; | |
436 | msg.msg_id = MICVNET_MSG_LINK_DOWN; | |
437 | micvnet_msg_send_msg(vnet_info, &msg); | |
438 | } | |
439 | ||
440 | static void | |
441 | micvnet_msg_recv_msg_link_down(struct micvnet_info *vnet_info) | |
442 | { | |
443 | atomic_set(&vnet_info->vi_state, MICVNET_STATE_BEGIN_UNINIT); | |
444 | ||
445 | if (vnet_info->link_down_initiator) | |
446 | wake_up_interruptible(&vnet_info->stop_waitq); | |
447 | else | |
448 | schedule_work(&vnet_info->vi_ws_stop); | |
449 | } | |
450 | ||
451 | static void | |
452 | micvnet_msg_send_link_up_msg(struct micvnet_info *vnet_info) | |
453 | { | |
454 | struct micvnet_msg msg; | |
455 | struct micvnet_msg_link_up | |
456 | *body = &msg.body.micvnet_msg_link_up; | |
457 | ||
458 | msg.msg_id = MICVNET_MSG_LINK_UP; | |
459 | body->vnet_driver_version = VNET_DRIVER_VERSION; | |
460 | micvnet_msg_send_msg(vnet_info, &msg); | |
461 | } | |
462 | ||
463 | static void | |
464 | micvnet_msg_recv_msg_link_up(struct micvnet_info *vnet_info, | |
465 | struct micvnet_msg_link_up *msg) | |
466 | { | |
467 | if (msg->vnet_driver_version != VNET_DRIVER_VERSION) { | |
468 | printk(KERN_ERR "%s: Error: vnet driver version mismatch: " | |
469 | "expected %d actual %lld\n" | |
470 | "Ensure that host and card modules are " | |
471 | "from the same build.\n", | |
472 | __func__, VNET_DRIVER_VERSION, | |
473 | msg->vnet_driver_version); | |
474 | return; | |
475 | } | |
476 | #ifdef HOST | |
477 | schedule_work(&vnet_info->vi_ws_start); | |
478 | #else | |
479 | micvnet_send_add_dma_buffer_messages(vnet_info); | |
480 | #endif | |
481 | } | |
482 | ||
483 | static void | |
484 | micvnet_msg_process_messages(struct micvnet_info *vnet_info) | |
485 | { | |
486 | struct micvnet_msg msg; | |
487 | ||
488 | #ifdef HOST | |
489 | micpm_get_reference(vnet_to_ctx(vnet_info), true); | |
490 | #endif | |
491 | while (!micvnet_msg_rb_read_msg(vnet_info, &msg)) { | |
492 | switch(msg.msg_id) { | |
493 | case MICVNET_MSG_ADD_DMA_BUFFER: | |
494 | micvnet_msg_recv_add_dma_buffer | |
495 | (vnet_info, | |
496 | &msg.body.micvnet_msg_add_dma_buffer); | |
497 | break; | |
498 | ||
499 | case MICVNET_MSG_DMA_COMPLETE: | |
500 | micvnet_msg_recv_dma_complete | |
501 | (vnet_info, | |
502 | &msg.body.micvnet_msg_dma_complete); | |
503 | break; | |
504 | ||
505 | case MICVNET_MSG_LINK_DOWN: | |
506 | micvnet_msg_recv_msg_link_down(vnet_info); | |
507 | break; | |
508 | ||
509 | case MICVNET_MSG_LINK_UP: | |
510 | micvnet_msg_recv_msg_link_up(vnet_info, | |
511 | &msg.body.micvnet_msg_link_up); | |
512 | break; | |
513 | ||
514 | default: | |
515 | printk(KERN_ERR "BUG: unknown vnet msg id: %lld\n", msg.msg_id); | |
516 | break; | |
517 | } | |
518 | } | |
519 | #ifdef HOST | |
520 | micpm_put_reference(vnet_to_ctx(vnet_info)); | |
521 | #endif | |
522 | } | |
523 | ||
524 | /*********************************************************** | |
525 | * Interrupts | |
526 | */ | |
527 | #ifdef HOST | |
528 | static int | |
529 | micvnet_host_doorbell_intr_handler(mic_ctx_t *mic_ctx, int doorbell) | |
530 | { | |
531 | struct micvnet_info *vnet_info; | |
532 | vnet_info = mic_ctx->bi_vethinfo; | |
533 | ||
534 | queue_work(vnet_info->vi_wq, &vnet_info->vi_ws_bh); | |
535 | return 0; | |
536 | } | |
537 | #else | |
538 | static irqreturn_t | |
539 | micvnet_host_intr_handler(int irq, void *data) | |
540 | { | |
541 | struct micvnet_info *vnet_info = data; | |
542 | queue_work(vnet_info->vi_wq, &vnet_info->vi_ws_bh); | |
543 | return IRQ_HANDLED; | |
544 | } | |
545 | #endif | |
546 | ||
547 | static void | |
548 | micvnet_intr_bh_handler(struct work_struct *work) | |
549 | { | |
550 | struct micvnet_info *vnet_info | |
551 | = container_of(work, struct micvnet_info, vi_ws_bh); | |
552 | ||
553 | micvnet_msg_process_messages(vnet_info); | |
554 | } | |
555 | ||
556 | #ifdef HOST | |
557 | static void | |
558 | micvnet_send_intr(struct micvnet_info *vnet_info) | |
559 | { | |
560 | mic_ctx_t *mic_ctx = vnet_info->mic_ctx; | |
561 | mic_send_vnet_intr(mic_ctx); | |
562 | } | |
563 | #else | |
564 | /* Ring host doorbell 3 interrupt */ | |
565 | static void | |
566 | micvnet_send_intr(struct micvnet_info *vnet_info) | |
567 | { | |
568 | uint32_t db_reg; | |
569 | ||
570 | /* Ring host doorbell 3 interrupt */ | |
571 | db_reg = readl(vnet_info->vi_sbox + SBOX_SDBIC3) | |
572 | | SBOX_SDBIC0_DBREQ_BIT; | |
573 | writel(db_reg, vnet_info->vi_sbox + SBOX_SDBIC3); | |
574 | } | |
575 | #endif | |
576 | ||
577 | /*********************************************************** | |
578 | * Net device ops and rtnl link ops | |
579 | */ | |
580 | /* | |
581 | Do nothing in ndo_open and ndo_stop. There are two reasons for this: | |
582 | 1. Since host and card side drivers are driver pairs, if ifconfig up or | |
583 | ifconfig down occurs on one side this needs to be communicated to the other | |
584 | side other side otherwise in the current implementation this can bring down | |
585 | the system. Ignoring ifconfig up or down avoids this issue. | |
586 | 2. For now, micvnet_init is called before the dma can be initialized. However, | |
587 | as soon as micvnet_init has been called and netdev has been created, the OS | |
588 | can invoke .ndo_open, which however requires the DMA to have been | |
589 | initialized. But DMA can not be initialized until later (at present after | |
590 | the card has booted). | |
591 | Therefore we ourselves call micvnet_start and micvnet_stop at appropriate | |
592 | times when we are ready for them. The only consequence is all packets till | |
593 | micvnet_start has been invoked will be dropped in ndo_start_xmit. | |
594 | */ | |
595 | ||
596 | /* Start callback */ | |
597 | static int | |
598 | micvnet_start_dev(struct net_device *dev) | |
599 | { | |
600 | struct micvnet_info *vnet_info = dev->ml_priv; | |
601 | ||
602 | /* Stop the queue till the state becomes LINKUP. The queue will be started when | |
603 | dma buffers are added in micvnet_msg_recv_add_dma_buffer(). Not doing this | |
604 | results in packets getting dropped till state is LINKUP. */ | |
605 | if (atomic_read(&vnet_info->vi_state) != MICVNET_STATE_LINKUP) | |
606 | netif_stop_queue(vnet_info->vi_netdev); | |
607 | ||
608 | return 0; | |
609 | } | |
610 | ||
611 | /* Stop callback */ | |
612 | static int | |
613 | micvnet_stop_dev(struct net_device *dev) | |
614 | { | |
615 | return 0; | |
616 | } | |
617 | ||
618 | static void | |
619 | micvnet_dma_cb_bh(struct work_struct *work) | |
620 | { | |
621 | struct micvnet_info | |
622 | *vnet_info = container_of(work, struct micvnet_info, vi_ws_dmacb); | |
623 | struct sched_node *snode; | |
624 | ||
625 | if (!atomic_read(&vnet_info->cnt_dma_complete)) | |
626 | return; | |
627 | ||
628 | do { | |
629 | spin_lock_bh(&vnet_info->vi_txlock); | |
630 | snode = list_entry((&vnet_info->vi_sched_skb)->next, | |
631 | struct sched_node, list); | |
632 | list_del(&snode->list); | |
633 | spin_unlock_bh(&vnet_info->vi_txlock); | |
634 | ||
635 | micvnet_msg_send_dma_complete_msg(vnet_info, snode); | |
636 | ||
637 | micvnet_dec_cnt_tx_pending(vnet_info); | |
638 | #ifdef HOST | |
639 | mic_ctx_unmap_single(vnet_to_ctx(vnet_info), | |
640 | snode->dma_src_phys, snode->dma_size); | |
641 | micpm_put_reference(vnet_to_ctx(vnet_info)); | |
642 | #endif | |
643 | kfree_skb(snode->skb); | |
644 | kfree(snode); | |
645 | ||
646 | } while (!atomic_dec_and_test(&vnet_info->cnt_dma_complete)); | |
647 | } | |
648 | ||
649 | static void | |
650 | micvnet_dma_completion_callback(uint64_t data) | |
651 | { | |
652 | struct micvnet_info *vnet_info = (struct micvnet_info *) data; | |
653 | ||
654 | atomic_inc(&vnet_info->cnt_dma_complete); | |
655 | ||
656 | queue_work(vnet_info->vi_wq, &vnet_info->vi_ws_dmacb); | |
657 | } | |
658 | ||
659 | static int | |
660 | micvnet_do_dma(struct micvnet_info *vnet_info, struct sched_node *snode) | |
661 | { | |
662 | uint64_t dma_src, dma_dst; | |
663 | int ret = 0; | |
664 | ||
665 | dma_src = snode->dma_src_phys; | |
666 | dma_dst = ALIGN(snode->dst_phys, DMA_ALIGNMENT); | |
667 | snode->dma_offset = (snode->skb->data - snode->skb_data_aligned) | |
668 | + (dma_dst - snode->dst_phys); | |
669 | if ((ret = request_dma_channel(vnet_info->dma_chan))) | |
670 | goto err_exit; | |
671 | ||
672 | ret = do_dma(vnet_info->dma_chan, | |
673 | DO_DMA_INTR, | |
674 | dma_src, | |
675 | dma_dst, | |
676 | snode->dma_size, | |
677 | &vnet_info->dma_cb); | |
678 | ||
679 | free_dma_channel(vnet_info->dma_chan); | |
680 | ||
681 | err_exit: | |
682 | return ret; | |
683 | } | |
684 | ||
685 | static int | |
686 | micvnet_schedule_dma(struct micvnet_info *vnet_info) | |
687 | { | |
688 | struct tx_node *tnode; | |
689 | struct sched_node *snode; | |
690 | struct dma_node *dnode; | |
691 | struct sk_buff *skb; | |
692 | int ret = 0; | |
693 | /* tnode */ | |
694 | spin_lock_bh(&vnet_info->vi_txlock); | |
695 | BUG_ON(list_empty(&vnet_info->vi_tx_skb)); | |
696 | tnode = list_entry((&vnet_info->vi_tx_skb)->next, | |
697 | struct tx_node, list); | |
698 | list_del(&tnode->list); | |
699 | spin_unlock_bh(&vnet_info->vi_txlock); | |
700 | skb = tnode->skb; | |
701 | kfree(tnode); | |
702 | ||
703 | #ifdef HOST | |
704 | if ((ret = micpm_get_reference(vnet_to_ctx(vnet_info), true))) | |
705 | goto err_exit_no_dec_node_refcnt; | |
706 | #endif | |
707 | ||
708 | /* dnode */ | |
709 | spin_lock(&vnet_info->vi_rxlock); | |
710 | BUG_ON(list_empty(&vnet_info->vi_dma_buf)); | |
711 | dnode = list_entry((&vnet_info->vi_dma_buf)->next, | |
712 | struct dma_node, list); | |
713 | spin_unlock(&vnet_info->vi_rxlock); | |
714 | if (dnode->size < skb->len + 3 * DMA_ALIGNMENT) { | |
715 | ret = -ENOMEM; | |
716 | goto err_exit; | |
717 | } | |
718 | ||
719 | /* snode */ | |
720 | if (!(snode = kmalloc(sizeof(*snode), GFP_KERNEL))) { | |
721 | ret = -ENOMEM; | |
722 | goto err_exit; | |
723 | } | |
724 | snode->skb = skb; | |
725 | snode->dst_phys = dnode->phys; | |
726 | snode->skb_data_aligned | |
727 | = (unsigned char *) ((uint64_t) skb->data & ~(DMA_ALIGNMENT - 1)); | |
728 | snode->dma_size | |
729 | = ALIGN((skb->len + (skb->data - snode->skb_data_aligned)), | |
730 | DMA_ALIGNMENT); | |
731 | #ifdef HOST | |
732 | snode->dma_src_phys = mic_ctx_map_single(vnet_to_ctx(vnet_info), | |
733 | snode->skb_data_aligned, | |
734 | snode->dma_size); | |
735 | if (mic_map_error(snode->dma_src_phys)) { | |
736 | kfree(snode); | |
737 | ret = -ENOMEM; | |
738 | goto err_exit; | |
739 | } | |
740 | #else | |
741 | snode->dma_src_phys = virt_to_phys(snode->skb_data_aligned); | |
742 | #endif | |
743 | ||
744 | if ((ret = micvnet_do_dma(vnet_info, snode))) { | |
745 | #ifdef HOST | |
746 | mic_ctx_unmap_single(vnet_to_ctx(vnet_info), | |
747 | snode->dma_src_phys, snode->dma_size); | |
748 | #endif | |
749 | kfree(snode); | |
750 | goto err_exit; | |
751 | } | |
752 | ||
753 | /* Update snode/dnode lists only after all operations have successfully | |
754 | completed and no further errors are possible */ | |
755 | spin_lock_bh(&vnet_info->vi_txlock); | |
756 | list_add_tail(&snode->list, &vnet_info->vi_sched_skb); | |
757 | spin_unlock_bh(&vnet_info->vi_txlock); | |
758 | ||
759 | spin_lock(&vnet_info->vi_rxlock); | |
760 | list_del(&dnode->list); | |
761 | spin_unlock(&vnet_info->vi_rxlock); | |
762 | list_obj_free(&vnet_info->dnode_list); | |
763 | ||
764 | vnet_info->vi_netdev->stats.tx_packets++; | |
765 | vnet_info->vi_netdev->stats.tx_bytes += skb->len; | |
766 | ||
767 | return ret; | |
768 | ||
769 | err_exit: | |
770 | #ifdef HOST | |
771 | micpm_put_reference(vnet_to_ctx(vnet_info)); | |
772 | err_exit_no_dec_node_refcnt: | |
773 | #endif | |
774 | micvnet_dec_cnt_tx_pending(vnet_info); | |
775 | atomic_inc(&vnet_info->cnt_dma_buf_avail); | |
776 | micvnet_wake_queue(vnet_info); | |
777 | skb->dev->stats.tx_dropped++; | |
778 | kfree_skb(skb); | |
779 | return ret; | |
780 | } | |
781 | ||
782 | static void | |
783 | micvnet_schedule_dmas(struct work_struct *work) | |
784 | { | |
785 | struct micvnet_info *vnet_info | |
786 | = container_of(work, struct micvnet_info, vi_ws_tx); | |
787 | volatile bool tx_skb_list_empty; | |
788 | while (1) { | |
789 | spin_lock_bh(&vnet_info->vi_txlock); | |
790 | tx_skb_list_empty = list_empty(&vnet_info->vi_tx_skb); | |
791 | spin_unlock_bh(&vnet_info->vi_txlock); | |
792 | if (tx_skb_list_empty) | |
793 | break; | |
794 | ||
795 | micvnet_schedule_dma(vnet_info); | |
796 | } | |
797 | } | |
798 | ||
799 | int | |
800 | micvnet_xmit(struct sk_buff *skb, struct net_device *dev) | |
801 | { | |
802 | struct micvnet_info *vnet_info = (struct micvnet_info*)dev->ml_priv; | |
803 | struct tx_node *tnode; | |
804 | if (!vnet_info || !atomic_read(&vnet_info->cnt_dma_buf_avail)){ | |
805 | goto err_exit; | |
806 | } | |
807 | ||
808 | if (!(tnode = kmalloc(sizeof(*tnode), GFP_ATOMIC))) | |
809 | goto err_exit; | |
810 | tnode->skb = skb; | |
811 | ||
812 | spin_lock(&vnet_info->vi_txlock); | |
813 | if (atomic_read(&vnet_info->vi_state) != MICVNET_STATE_LINKUP) | |
814 | goto err_exit_unlock; | |
815 | list_add_tail(&tnode->list, &vnet_info->vi_tx_skb); | |
816 | atomic_inc(&vnet_info->cnt_tx_pending); | |
817 | spin_unlock(&vnet_info->vi_txlock); | |
818 | ||
819 | queue_work(vnet_info->vi_wq, &vnet_info->vi_ws_tx); | |
820 | ||
821 | if (atomic_dec_and_test(&vnet_info->cnt_dma_buf_avail)) | |
822 | netif_stop_queue(vnet_info->vi_netdev); | |
823 | ||
824 | return NETDEV_TX_OK; | |
825 | ||
826 | err_exit_unlock: | |
827 | kfree(tnode); | |
828 | spin_unlock(&vnet_info->vi_txlock); | |
829 | err_exit: | |
830 | kfree_skb(skb); | |
831 | dev->stats.tx_dropped++; | |
832 | return NETDEV_TX_OK; | |
833 | } | |
834 | ||
835 | #if LINUX_VERSION_CODE < KERNEL_VERSION(3,2,0) | |
836 | static void | |
837 | micvnet_multicast_list(struct net_device *dev) | |
838 | { | |
839 | } | |
840 | #endif | |
841 | ||
842 | static int | |
843 | micvnet_set_address(struct net_device *dev, void *p) | |
844 | { | |
845 | struct sockaddr *sa = p; | |
846 | ||
847 | if (!is_valid_ether_addr(sa->sa_data)) | |
848 | return -EADDRNOTAVAIL; | |
849 | ||
850 | memcpy(dev->dev_addr, sa->sa_data, ETH_ALEN); | |
851 | return 0; | |
852 | } | |
853 | ||
854 | #define MIN_MTU 68 | |
855 | #define MAX_MTU MICVNET_MAX_MTU | |
856 | ||
857 | static int | |
858 | micvnet_change_mtu(struct net_device *dev, int new_mtu) | |
859 | { | |
860 | if (new_mtu < MIN_MTU || new_mtu > MAX_MTU) | |
861 | return -EINVAL; | |
862 | dev->mtu = new_mtu; | |
863 | return 0; | |
864 | } | |
865 | ||
866 | union serial { | |
867 | uint32_t regs[3]; | |
868 | char string[13]; | |
869 | }; | |
870 | ||
871 | void | |
872 | mic_get_serial_from_dbox(struct micvnet_info *vni, char *serialnum) | |
873 | { | |
874 | union serial serial; | |
875 | #ifdef HOST | |
876 | serial.regs[0] = DBOX_READ(vni->mic_ctx->mmio.va, DBOX_SWF1X0); | |
877 | serial.regs[1] = DBOX_READ(vni->mic_ctx->mmio.va, DBOX_SWF1X1); | |
878 | serial.regs[2] = DBOX_READ(vni->mic_ctx->mmio.va, DBOX_SWF1X2); | |
879 | #else | |
880 | serial.regs[0] = readl(vni->vi_dbox + DBOX_SWF1X0); | |
881 | serial.regs[1] = readl(vni->vi_dbox + DBOX_SWF1X1); | |
882 | serial.regs[2] = readl(vni->vi_dbox + DBOX_SWF1X2); | |
883 | #endif | |
884 | serial.string[12] = '\0'; | |
885 | strcpy(serialnum, serial.string); | |
886 | } | |
887 | ||
888 | int | |
889 | micvnet_setmac_from_serial(struct net_device *dev) | |
890 | { | |
891 | struct micvnet_info *vni = (struct micvnet_info *)dev->ml_priv; | |
892 | char serialnum[17]; | |
893 | int err; | |
894 | ||
895 | mic_get_serial_from_dbox(vni, serialnum); | |
896 | #ifdef HOST | |
897 | err = mic_get_mac_from_serial(serialnum, dev->dev_addr, 1); | |
898 | #else | |
899 | err = mic_get_mac_from_serial(serialnum, dev->dev_addr, 0); | |
900 | #endif | |
901 | return err; | |
902 | } | |
903 | ||
904 | static const struct net_device_ops micvnet_netdev_ops = { | |
905 | .ndo_open = micvnet_start_dev, | |
906 | .ndo_stop = micvnet_stop_dev, | |
907 | .ndo_start_xmit = micvnet_xmit, | |
908 | .ndo_validate_addr = eth_validate_addr, | |
909 | #if LINUX_VERSION_CODE < KERNEL_VERSION(3,2,0) | |
910 | .ndo_set_multicast_list = micvnet_multicast_list, | |
911 | #endif | |
912 | .ndo_set_mac_address = micvnet_set_address, | |
913 | .ndo_change_mtu = micvnet_change_mtu, | |
914 | }; | |
915 | ||
916 | static void | |
917 | micvnet_setup(struct net_device *dev) | |
918 | { | |
919 | ether_setup(dev); | |
920 | ||
921 | /* Initialize the device structure. */ | |
922 | dev->netdev_ops = &micvnet_netdev_ops; | |
cb91baf6 AT |
923 | dev->priv_destructor = free_netdev; |
924 | dev->needs_free_netdev = false; | |
800f879a AT |
925 | |
926 | /* Fill in device structure with ethernet-generic values. */ | |
927 | dev->mtu = MICVNET_MAX_MTU; | |
928 | dev->flags &= ~IFF_MULTICAST; | |
929 | } | |
930 | ||
931 | static struct rtnl_link_ops micvnet_link_ops __read_mostly = { | |
932 | .kind = "micvnet", | |
933 | .setup = micvnet_setup, | |
934 | }; | |
935 | ||
936 | /*********************************************************** | |
937 | * Vnet init/deinit | |
938 | */ | |
939 | static int | |
940 | micvnet_init_hw_regs(struct micvnet_info *vnet_info) | |
941 | { | |
942 | #ifdef HOST | |
943 | mic_ctx_t *mic_ctx = vnet_info->mic_ctx; | |
944 | ||
945 | vnet_info->vi_pdev = mic_ctx->bi_pdev; | |
946 | vnet_info->vi_sbox = (uint8_t *)((unsigned long) mic_ctx->mmio.va + | |
947 | HOST_SBOX_BASE_ADDRESS); | |
948 | vnet_info->vi_scratch14 | |
949 | = (uint32_t *)((unsigned long)mic_ctx->mmio.va + | |
950 | HOST_SBOX_BASE_ADDRESS + SBOX_SCRATCH14); | |
951 | #else | |
952 | vnet_info->vi_sbox = ioremap_nocache(SBOX_BASE, SBOX_MMIO_LENGTH); | |
953 | vnet_info->vi_dbox = ioremap_nocache(DBOX_BASE, SBOX_MMIO_LENGTH); | |
954 | if (!vnet_info->vi_sbox) { | |
955 | printk(KERN_ERR "%s: NULL SBOX ptr\n", __func__); | |
956 | return -ENOMEM; | |
957 | } | |
958 | vnet_info->vi_scratch14 | |
959 | = (uint32_t *)(vnet_info->vi_sbox + SBOX_SCRATCH14); | |
960 | #endif | |
961 | return 0; | |
962 | } | |
963 | ||
964 | static void | |
965 | micvnet_deinit_hw_regs(struct micvnet_info *vnet_info) | |
966 | { | |
967 | #ifndef HOST | |
968 | iounmap(vnet_info->vi_sbox); | |
969 | iounmap(vnet_info->vi_dbox); | |
970 | #endif | |
971 | } | |
972 | ||
973 | static int | |
974 | micvnet_init_interrupts(struct micvnet_info *vnet_info) | |
975 | { | |
976 | mic_ctx_t *mic_ctx = vnet_info->mic_ctx; | |
977 | int ret = 0; | |
978 | ||
979 | spin_lock_init(&vnet_info->vi_txlock); | |
980 | spin_lock_init(&vnet_info->vi_rxlock); | |
981 | ||
982 | snprintf(vnet_info->vi_wqname, sizeof(vnet_info->vi_wqname), | |
983 | "VNET WQ %d", mic_ctx->bi_id); | |
984 | ||
985 | if (!(vnet_info->vi_wq = | |
986 | __mic_create_singlethread_workqueue(vnet_info->vi_wqname))) { | |
987 | printk(KERN_ERR "%s: create_singlethread_workqueue\n", __func__); | |
988 | return -ENOMEM; | |
989 | } | |
990 | init_waitqueue_head(&vnet_info->stop_waitq); | |
991 | ||
992 | INIT_WORK(&vnet_info->vi_ws_bh, micvnet_intr_bh_handler); | |
993 | INIT_WORK(&vnet_info->vi_ws_tx, micvnet_schedule_dmas); | |
994 | INIT_WORK(&vnet_info->vi_ws_dmacb, micvnet_dma_cb_bh); | |
995 | INIT_WORK(&vnet_info->vi_ws_link_down, micvnet_msg_send_link_down_msg); | |
996 | INIT_WORK(&vnet_info->vi_ws_stop, micvnet_stop_ws); | |
997 | INIT_WORK(&vnet_info->vi_ws_start, micvnet_start_ws); | |
998 | #ifdef HOST | |
999 | if ((ret = mic_reg_irqhandler(mic_ctx, 3, "Host DoorBell 3", | |
1000 | micvnet_host_doorbell_intr_handler))) { | |
1001 | #else | |
1002 | if ((ret = request_irq(get_sbox_irq(VNET_SBOX_INT_IDX), | |
1003 | micvnet_host_intr_handler, IRQF_DISABLED, | |
1004 | "vnet intr", vnet_info))) { | |
1005 | #endif | |
1006 | printk(KERN_ERR "%s: interrupt registration failed\n", __func__); | |
1007 | goto err_exit_destroy_workqueue; | |
1008 | } | |
1009 | return 0; | |
1010 | ||
1011 | err_exit_destroy_workqueue: | |
1012 | destroy_workqueue(vnet_info->vi_wq); | |
1013 | return ret; | |
1014 | } | |
1015 | ||
1016 | static void | |
1017 | micvnet_deinit_interrupts(struct micvnet_info *vnet_info) | |
1018 | { | |
1019 | #ifdef HOST | |
1020 | mic_unreg_irqhandler(vnet_info->mic_ctx, 3, "Host DoorBell 3"); | |
1021 | #else | |
1022 | free_irq(get_sbox_irq(VNET_SBOX_INT_IDX), vnet_info); | |
1023 | #endif | |
1024 | destroy_workqueue(vnet_info->vi_wq); | |
1025 | } | |
1026 | ||
1027 | ||
1028 | static int | |
1029 | micvnet_init_netdev(struct micvnet_info *vnet_info) | |
1030 | { | |
1031 | struct net_device *dev_vnet; | |
1032 | int ret = 0; | |
1033 | ||
1034 | ||
1035 | #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0)) | |
1036 | if ((dev_vnet = (struct net_device *)alloc_netdev(sizeof(struct micvnet_info), "mic%d", | |
1037 | NET_NAME_UNKNOWN, micvnet_setup)) == NULL) { | |
1038 | #else | |
1039 | if ((dev_vnet = (struct net_device *)alloc_netdev(sizeof(struct micvnet_info), "mic%d", | |
1040 | micvnet_setup)) == NULL) { | |
1041 | #endif | |
1042 | printk(KERN_ERR "%s: alloc_netdev failed\n", __func__); | |
1043 | return -ENOMEM; | |
1044 | } | |
1045 | ||
1046 | vnet_info->vi_netdev = dev_vnet; | |
1047 | dev_vnet->ml_priv = vnet_info; | |
1048 | ||
1049 | if (micvnet_setmac_from_serial(dev_vnet)) | |
1050 | random_ether_addr(dev_vnet->dev_addr); | |
1051 | ||
1052 | dev_vnet->rtnl_link_ops = &micvnet_link_ops; | |
1053 | ||
1054 | if ((ret = register_netdev(dev_vnet)) < 0) { | |
1055 | printk(KERN_ERR "%s: register_netdev failed %d\n", __func__, ret); | |
1056 | free_netdev(dev_vnet); | |
1057 | return ret; | |
1058 | } | |
1059 | ||
1060 | return 0; | |
1061 | } | |
1062 | ||
1063 | static int | |
1064 | micvnet_init_msg_rings(struct micvnet_info *vnet_info) | |
1065 | { | |
1066 | #ifdef HOST | |
1067 | vnet_info->vi_qp.tx = &vnet_info->vi_rp.rb_tx; | |
1068 | vnet_info->vi_qp.rx = &vnet_info->vi_rp.rb_rx; | |
1069 | micvnet_reset_msg_rings(vnet_info); | |
1070 | ||
1071 | vnet_info->vi_rp_phys = mic_ctx_map_single(vnet_to_ctx(vnet_info), | |
1072 | &vnet_info->vi_rp, | |
1073 | sizeof(vnet_info->vi_rp)); | |
1074 | if (mic_map_error(vnet_info->vi_rp_phys)) { | |
1075 | printk(KERN_ERR "%s: mic_map_error failed\n", __func__); | |
1076 | return -ENOMEM; | |
1077 | } | |
1078 | #else | |
1079 | if (!(vnet_info->vi_rp_phys = vnet_addr)) { | |
1080 | printk(KERN_ERR "%s: null vnet_addr\n", __func__); | |
1081 | return -ENOMEM; | |
1082 | } | |
1083 | vnet_info->ring_ptr | |
1084 | = ioremap_nocache(vnet_info->vi_rp_phys, | |
1085 | sizeof(struct micvnet_msg_ring_pair)); | |
1086 | if (!vnet_info->ring_ptr) { | |
1087 | printk(KERN_ERR "%s: NULL ring ptr\n", __func__); | |
1088 | return -ENOMEM; | |
1089 | } | |
1090 | vnet_info->vi_qp.tx = &vnet_info->ring_ptr->rb_rx; | |
1091 | vnet_info->vi_qp.rx = &vnet_info->ring_ptr->rb_tx; | |
1092 | #endif | |
1093 | return 0; | |
1094 | } | |
1095 | ||
1096 | static void | |
1097 | micvnet_deinit_msg_rings(struct micvnet_info *vnet_info) | |
1098 | { | |
1099 | #ifdef HOST | |
1100 | mic_ctx_unmap_single(vnet_to_ctx(vnet_info), | |
1101 | vnet_info->vi_rp_phys, sizeof(vnet_info->vi_rp)); | |
1102 | #else | |
1103 | iounmap(vnet_info->ring_ptr); | |
1104 | #endif | |
1105 | } | |
1106 | ||
1107 | static int | |
1108 | micvnet_init_lists(struct micvnet_info *vnet_info) | |
1109 | { | |
1110 | int ret; | |
1111 | if ((ret = list_obj_list_init(VNET_MAX_SKBS, sizeof(struct dma_node), | |
1112 | &vnet_info->dnode_list))) | |
1113 | return ret; | |
1114 | ||
1115 | INIT_LIST_HEAD(&vnet_info->vi_rx_skb); | |
1116 | INIT_LIST_HEAD(&vnet_info->vi_dma_buf); | |
1117 | INIT_LIST_HEAD(&vnet_info->vi_tx_skb); | |
1118 | INIT_LIST_HEAD(&vnet_info->vi_sched_skb); | |
1119 | return 0; | |
1120 | } | |
1121 | ||
1122 | static void | |
1123 | micvnet_deinit_lists(struct micvnet_info *vnet_info) | |
1124 | { | |
1125 | struct list_head *pos, *tmpq; | |
1126 | struct rx_node *rnode; | |
1127 | struct tx_node *tnode; | |
1128 | struct dma_node *dnode; | |
1129 | struct sched_node *snode; | |
1130 | ||
1131 | list_for_each_safe(pos, tmpq, &vnet_info->vi_rx_skb) { | |
1132 | rnode = list_entry(pos, struct rx_node, list); | |
1133 | list_del(&rnode->list); | |
1134 | #ifdef HOST | |
1135 | mic_ctx_unmap_single(vnet_to_ctx(vnet_info), | |
1136 | rnode->phys, rnode->size); | |
1137 | #endif | |
1138 | kfree_skb(rnode->skb); | |
1139 | kfree(rnode); | |
1140 | } | |
1141 | ||
1142 | list_for_each_safe(pos, tmpq, &vnet_info->vi_dma_buf) { | |
1143 | dnode = list_entry(pos, struct dma_node, list); | |
1144 | list_del(&dnode->list); | |
1145 | list_obj_free(&vnet_info->dnode_list); | |
1146 | } | |
1147 | ||
1148 | list_for_each_safe(pos, tmpq, &vnet_info->vi_tx_skb) { | |
1149 | tnode = list_entry(pos, struct tx_node, list); | |
1150 | list_del(&tnode->list); | |
1151 | kfree_skb(tnode->skb); | |
1152 | kfree(tnode); | |
1153 | } | |
1154 | ||
1155 | list_for_each_safe(pos, tmpq, &vnet_info->vi_sched_skb) { | |
1156 | snode = list_entry(pos, struct sched_node, list); | |
1157 | list_del(&snode->list); | |
1158 | #ifdef HOST | |
1159 | mic_ctx_unmap_single(vnet_to_ctx(vnet_info), snode->dma_src_phys, | |
1160 | snode->dma_size); | |
1161 | micpm_put_reference(vnet_to_ctx(vnet_info)); | |
1162 | #endif | |
1163 | kfree_skb(snode->skb); | |
1164 | kfree(snode); | |
1165 | } | |
1166 | ||
1167 | list_obj_list_deinit(&vnet_info->dnode_list); | |
1168 | } | |
1169 | static int | |
1170 | micvnet_init_dma(struct micvnet_info *vnet_info) | |
1171 | { | |
1172 | mic_ctx_t *mic_ctx = vnet_info->mic_ctx; | |
1173 | int ret; | |
1174 | ||
1175 | /* Note: open_dma_device must use mic_ctx->dma_handle since that is | |
1176 | used in the isr */ | |
1177 | #ifdef HOST | |
1178 | if (micpm_get_reference(mic_ctx, true) != 0) { | |
1179 | printk(KERN_ERR "%s: micpm_get_reference failed\n", __func__); | |
1180 | return -ENODEV; | |
1181 | } | |
1182 | ||
1183 | if ((ret = open_dma_device(mic_ctx->bi_id + 1, | |
1184 | mic_ctx->mmio.va + HOST_SBOX_BASE_ADDRESS, | |
1185 | &mic_ctx->dma_handle))) { | |
1186 | printk(KERN_ERR "%s: open_dma_device failed\n", __func__); | |
1187 | micpm_put_reference(mic_ctx); | |
1188 | return ret; | |
1189 | } | |
1190 | micpm_put_reference(mic_ctx); | |
1191 | #else | |
1192 | if ((ret = open_dma_device(0, 0, &mic_ctx->dma_handle))) { | |
1193 | printk(KERN_ERR "%s: open_dma_device failed\n", __func__); | |
1194 | return ret; | |
1195 | } | |
1196 | #endif | |
1197 | ||
1198 | vnet_info->dma_handle = mic_ctx->dma_handle; | |
1199 | ||
1200 | if ((ret = allocate_dma_channel(vnet_info->dma_handle, | |
1201 | &vnet_info->dma_chan))) { | |
1202 | printk(KERN_ERR "%s: allocate_dma_channel failed\n", __func__); | |
1203 | goto err_exit_close_dma; | |
1204 | } | |
1205 | free_dma_channel(vnet_info->dma_chan); | |
1206 | vnet_info->dma_cb.dma_completion_func = micvnet_dma_completion_callback; | |
1207 | vnet_info->dma_cb.cb_cookie = (uint64_t) vnet_info; | |
1208 | atomic_set(&vnet_info->cnt_dma_complete, 0); | |
1209 | atomic_set(&vnet_info->cnt_dma_buf_avail, 0); | |
1210 | vnet_info->link_down_initiator = false; | |
1211 | atomic_set(&vnet_info->cnt_tx_pending, 0); | |
1212 | return 0; | |
1213 | ||
1214 | err_exit_close_dma: | |
1215 | close_dma_device(mic_ctx->bi_id + 1, &vnet_info->dma_handle); | |
1216 | return ret; | |
1217 | } | |
1218 | ||
1219 | static void | |
1220 | micvnet_deinit_dma(struct micvnet_info *vnet_info) | |
1221 | { | |
1222 | mic_ctx_t *mic_ctx = vnet_info->mic_ctx; | |
1223 | ||
1224 | close_dma_device(mic_ctx->bi_id + 1, &vnet_info->dma_handle); | |
1225 | } | |
1226 | static int | |
1227 | micvnet_alloc_rx_node(struct micvnet_info *vnet_info, struct rx_node **node) | |
1228 | { | |
1229 | struct rx_node *rnode; | |
1230 | ||
1231 | if (!(rnode = kmalloc(sizeof(*rnode), GFP_KERNEL))) | |
1232 | return -ENOMEM; | |
1233 | ||
1234 | rnode->size = vnet_info->vi_netdev->mtu + 3 * DMA_ALIGNMENT + ETH_HLEN; | |
1235 | ||
1236 | if (!(rnode->skb = dev_alloc_skb(rnode->size))) { | |
1237 | kfree(rnode); | |
1238 | return -ENOMEM; | |
1239 | } | |
1240 | ||
1241 | #ifdef HOST | |
1242 | rnode->phys = mic_ctx_map_single(vnet_to_ctx(vnet_info), | |
1243 | rnode->skb->data, rnode->size); | |
1244 | if (mic_map_error(rnode->phys)) { | |
1245 | kfree_skb(rnode->skb); | |
1246 | kfree(rnode); | |
1247 | return -ENOMEM; | |
1248 | } | |
1249 | #else | |
1250 | rnode->phys = virt_to_phys(rnode->skb->data); | |
1251 | #endif | |
1252 | ||
1253 | *node = rnode; | |
1254 | ||
1255 | return 0; | |
1256 | } | |
1257 | ||
1258 | static int | |
1259 | micvnet_init_rx_skb_send_msg(struct micvnet_info *vnet_info) | |
1260 | { | |
1261 | struct rx_node *rnode; | |
1262 | int ret = 0; | |
1263 | ||
1264 | if (unlikely(ret = micvnet_alloc_rx_node(vnet_info, &rnode))) | |
1265 | return ret; | |
1266 | ||
1267 | list_add_tail(&rnode->list, &vnet_info->vi_rx_skb); | |
1268 | ||
1269 | micvnet_msg_send_add_dma_buffer_msg(vnet_info, rnode); | |
1270 | ||
1271 | return 0; | |
1272 | } | |
1273 | ||
1274 | static int | |
1275 | micvnet_init_rx_skbs(struct micvnet_info *vnet_info) | |
1276 | { | |
1277 | struct rx_node *rnode; | |
1278 | int i, ret = 0; | |
1279 | ||
1280 | ||
1281 | if ( (vnet_num_buffers <= 0) || (vnet_num_buffers > VNET_MAX_SKBS) ) | |
1282 | vnet_num_buffers = VNET_MAX_SKBS; | |
1283 | ||
1284 | for (i = 0; i < vnet_num_buffers; i++) { | |
1285 | if (unlikely(ret = micvnet_alloc_rx_node(vnet_info, &rnode))) | |
1286 | return ret; | |
1287 | ||
1288 | list_add_tail(&rnode->list, &vnet_info->vi_rx_skb); | |
1289 | } | |
1290 | ||
1291 | return ret; | |
1292 | } | |
1293 | ||
1294 | static void | |
1295 | micvnet_send_add_dma_buffer_messages(struct micvnet_info *vnet_info) | |
1296 | { | |
1297 | struct rx_node *rnode; | |
1298 | struct list_head *pos; | |
1299 | ||
1300 | list_for_each(pos, &vnet_info->vi_rx_skb) { | |
1301 | rnode = list_entry(pos, struct rx_node, list); | |
1302 | micvnet_msg_send_add_dma_buffer_msg(vnet_info, rnode); | |
1303 | } | |
1304 | } | |
1305 | ||
1306 | static void | |
1307 | micvnet_initiate_link_down(struct micvnet_info *vnet_info) | |
1308 | { | |
1309 | int ret; | |
1310 | netif_tx_disable(vnet_info->vi_netdev); | |
1311 | spin_lock_bh(&vnet_info->vi_txlock); | |
1312 | atomic_set(&vnet_info->vi_state, MICVNET_STATE_LINK_DOWN); | |
1313 | spin_unlock_bh(&vnet_info->vi_txlock); | |
1314 | ||
1315 | /* This wait precludes this function to be called from the context of | |
1316 | * the vnet wq thread */ | |
1317 | ret = wait_event_interruptible_timeout( | |
1318 | vnet_info->stop_waitq, | |
1319 | (atomic_read(&vnet_info->cnt_tx_pending) == 0), | |
1320 | STOP_WAIT_TIMEOUT); | |
1321 | if (!ret) | |
1322 | printk(KERN_ERR "%s timeout waiting for Tx dma buffers to drain\n", __func__); | |
1323 | /* To avoid introducing a lock in micvnet_msg_send_msg() send the | |
1324 | * LINK_DOWN message from vnet wq thread context. LINK_DOWN will be the | |
1325 | * LAST message sent. */ | |
1326 | queue_work(vnet_info->vi_wq, &vnet_info->vi_ws_link_down); | |
1327 | } | |
1328 | ||
1329 | static void | |
1330 | micvnet_stop_deinit(struct micvnet_info *vnet_info) | |
1331 | { | |
1332 | flush_workqueue(vnet_info->vi_wq); | |
1333 | atomic_set(&vnet_info->vi_state, MICVNET_STATE_UNINITIALIZED); | |
1334 | ||
1335 | micvnet_deinit_dma(vnet_info); | |
1336 | micvnet_deinit_lists(vnet_info); | |
1337 | #ifdef HOST | |
1338 | micvnet_reset_msg_rings(vnet_info); | |
1339 | #endif | |
1340 | atomic_dec(&micvnet.lv_active_clients); | |
1341 | } | |
1342 | ||
1343 | int | |
1344 | micvnet_probe(mic_ctx_t *mic_ctx) | |
1345 | { | |
1346 | struct micvnet_info *vnet_info; | |
1347 | int ret = 0; | |
1348 | ||
1349 | mic_ctx->bi_vethinfo = NULL; | |
1350 | ||
1351 | if (!micvnet.created) | |
1352 | return 1; | |
1353 | ||
1354 | if (!(vnet_info = kzalloc(sizeof(struct micvnet_info), GFP_KERNEL))) { | |
1355 | printk(KERN_ERR "%s: vnet_info alloc failed\n", __func__); | |
1356 | return -ENOMEM; | |
1357 | } | |
1358 | ||
1359 | mic_ctx->bi_vethinfo = vnet_info; | |
1360 | vnet_info->mic_ctx = mic_ctx; | |
1361 | if ((ret = micvnet_init_hw_regs(vnet_info))) | |
1362 | goto err_exit_free_vnet_info; | |
1363 | if ((ret = micvnet_init_msg_rings(vnet_info))) | |
1364 | goto err_exit_deinit_hw_regs; | |
1365 | if ((ret = micvnet_init_interrupts(vnet_info))) | |
1366 | goto err_exit_deinit_msg_rings; | |
1367 | if ((ret = micvnet_init_netdev(vnet_info))) | |
1368 | goto err_exit_deinit_interrupts; | |
1369 | ||
1370 | atomic_set(&vnet_info->vi_state, MICVNET_STATE_UNINITIALIZED); | |
1371 | return 0; | |
1372 | ||
1373 | err_exit_deinit_interrupts: | |
1374 | micvnet_deinit_interrupts(vnet_info); | |
1375 | err_exit_deinit_msg_rings: | |
1376 | micvnet_deinit_msg_rings(vnet_info); | |
1377 | err_exit_deinit_hw_regs: | |
1378 | micvnet_deinit_hw_regs(vnet_info); | |
1379 | err_exit_free_vnet_info: | |
1380 | kfree(vnet_info); | |
1381 | ||
1382 | return ret; | |
1383 | } | |
1384 | ||
1385 | void | |
1386 | micvnet_remove(mic_ctx_t *mic_ctx) | |
1387 | { | |
1388 | struct micvnet_info | |
1389 | *vnet_info = (struct micvnet_info *) mic_ctx->bi_vethinfo; | |
1390 | ||
1391 | if (!vnet_info) | |
1392 | return; | |
1393 | ||
1394 | micvnet_stop(mic_ctx); | |
1395 | ||
1396 | vnet_info->vi_netdev->ml_priv = NULL; | |
1397 | ||
1398 | micvnet_deinit_interrupts(vnet_info); | |
1399 | micvnet_deinit_msg_rings(vnet_info); | |
1400 | micvnet_deinit_hw_regs(vnet_info); | |
1401 | ||
1402 | mic_ctx->bi_vethinfo = NULL; | |
1403 | ||
1404 | kfree(vnet_info); | |
1405 | } | |
1406 | ||
1407 | int | |
1408 | micvnet_execute_start(struct micvnet_info *vnet_info) | |
1409 | { | |
1410 | int ret = 0; | |
1411 | ||
1412 | if (!vnet_info) { | |
1413 | printk(KERN_ERR "%s: vnet_info is NULL\n", __func__); | |
1414 | return 1; | |
1415 | } | |
1416 | ||
1417 | if (atomic_cmpxchg(&vnet_info->vi_state, MICVNET_STATE_UNINITIALIZED, | |
1418 | MICVNET_STATE_TRANSITIONING) != MICVNET_STATE_UNINITIALIZED) { | |
1419 | printk(KERN_ERR "%s: wrong vnet state %d\n", __func__, | |
1420 | atomic_read(&vnet_info->vi_state)); | |
1421 | return 1; | |
1422 | } | |
1423 | ||
1424 | if ((ret = micvnet_init_lists(vnet_info))) | |
1425 | goto err_exit; | |
1426 | if ((ret = micvnet_init_dma(vnet_info))) | |
1427 | goto err_exit_deinit_lists; | |
1428 | if ((ret = micvnet_init_rx_skbs(vnet_info))) { | |
1429 | printk(KERN_ERR "%s: micvnet_init_rx_skbs failed\n", __func__); | |
1430 | goto err_exit_deinit_dma; | |
1431 | } | |
1432 | ||
1433 | memset(&vnet_info->vi_netdev->stats, 0, sizeof(vnet_info->vi_netdev->stats)); | |
1434 | atomic_inc(&micvnet.lv_active_clients); | |
1435 | atomic_set(&vnet_info->vi_state, MICVNET_STATE_LINKUP); | |
1436 | ||
1437 | micvnet_msg_send_link_up_msg(vnet_info); | |
1438 | #ifdef HOST | |
1439 | micvnet_send_add_dma_buffer_messages(vnet_info); | |
1440 | #else | |
1441 | writel(MICVNET_CARD_UP_MAGIC, vnet_info->vi_scratch14); | |
1442 | /* Card adds DMA buffers to host after receiving MICVNET_MSG_LINK_UP */ | |
1443 | #endif | |
1444 | return 0; | |
1445 | ||
1446 | err_exit_deinit_dma: | |
1447 | micvnet_deinit_dma(vnet_info); | |
1448 | err_exit_deinit_lists: | |
1449 | /* RX SKB's are deallocated in micvnet_deinit_lists() */ | |
1450 | micvnet_deinit_lists(vnet_info); | |
1451 | err_exit: | |
1452 | atomic_set(&vnet_info->vi_state, MICVNET_STATE_UNINITIALIZED); | |
1453 | return ret; | |
1454 | } | |
1455 | ||
1456 | static void | |
1457 | micvnet_start_ws(struct work_struct *work) | |
1458 | { | |
1459 | struct micvnet_info *vnet_info | |
1460 | = container_of(work, struct micvnet_info, vi_ws_start); | |
1461 | ||
1462 | micvnet_execute_start(vnet_info); | |
1463 | } | |
1464 | ||
1465 | int micvnet_start(mic_ctx_t *mic_ctx) | |
1466 | { | |
1467 | #ifndef HOST | |
1468 | struct micvnet_info *vnet_info = (struct micvnet_info *) mic_ctx->bi_vethinfo; | |
1469 | micvnet_execute_start(vnet_info); | |
1470 | #endif | |
1471 | return 0; | |
1472 | } | |
1473 | ||
1474 | void | |
1475 | micvnet_execute_stop(struct micvnet_info *vnet_info) | |
1476 | { | |
1477 | int ret; | |
1478 | if (!vnet_info) | |
1479 | return; | |
1480 | ||
1481 | switch(atomic_read(&vnet_info->vi_state)) { | |
1482 | case MICVNET_STATE_LINKUP: | |
1483 | case MICVNET_STATE_BEGIN_UNINIT: | |
1484 | break; | |
1485 | default: | |
1486 | return; | |
1487 | } | |
1488 | ||
1489 | #ifdef HOST | |
1490 | if ((micpm_get_reference(vnet_to_ctx(vnet_info), true)) != 0) | |
1491 | goto exit; | |
1492 | #endif | |
1493 | micvnet_initiate_link_down(vnet_info); | |
1494 | if (vnet_info->link_down_initiator && !(vnet_info->mic_ctx->state == MIC_SHUTDOWN && vnet_info->mic_ctx->sdbic1)){ | |
1495 | ret = wait_event_interruptible_timeout( | |
1496 | vnet_info->stop_waitq, | |
1497 | (atomic_read(&vnet_info->vi_state) == MICVNET_STATE_BEGIN_UNINIT), | |
1498 | STOP_WAIT_TIMEOUT); | |
1499 | if (!ret) | |
1500 | printk(KERN_ERR "%s: timeout waiting for link down message response\n", __func__); | |
1501 | } | |
1502 | ||
1503 | #ifdef HOST | |
1504 | micpm_put_reference(vnet_to_ctx(vnet_info)); | |
1505 | exit: | |
1506 | #endif | |
1507 | micvnet_stop_deinit(vnet_info); | |
1508 | } | |
1509 | ||
1510 | void | |
1511 | micvnet_stop(mic_ctx_t *mic_ctx) | |
1512 | { | |
1513 | struct micvnet_info *vnet_info = (struct micvnet_info *) mic_ctx->bi_vethinfo; | |
1514 | ||
1515 | vnet_info->link_down_initiator = true; | |
1516 | micvnet_execute_stop(vnet_info); | |
1517 | } | |
1518 | ||
1519 | static void | |
1520 | micvnet_stop_ws(struct work_struct *work) | |
1521 | { | |
1522 | struct micvnet_info *vnet_info | |
1523 | = container_of(work, struct micvnet_info, vi_ws_stop); | |
1524 | ||
1525 | vnet_info->link_down_initiator = false; | |
1526 | micvnet_execute_stop(vnet_info); | |
1527 | } | |
1528 | ||
1529 | #if !defined(WINDOWS) && defined(HOST) | |
1530 | static ssize_t | |
1531 | show_vnet(struct device *dev, struct device_attribute *attr, char *buf); | |
1532 | DEVICE_ATTR(vnet, S_IRUGO, show_vnet, NULL); | |
1533 | ||
1534 | static ssize_t | |
1535 | show_vnet(struct device *dev, struct device_attribute *attr, char *buf) | |
1536 | { | |
1537 | return snprintf(buf, PAGE_SIZE, "Number of active vnet clients: %d\n", | |
1538 | atomic_read(&micvnet.lv_active_clients)); | |
1539 | } | |
1540 | #endif | |
1541 | ||
1542 | int | |
1543 | micvnet_init(struct device *dev) | |
1544 | { | |
1545 | int ret = 0; | |
1546 | ||
1547 | micvnet.created = 0; | |
1548 | atomic_set(&micvnet.lv_active_clients, 0); | |
1549 | ||
1550 | if ((ret = rtnl_link_register(&micvnet_link_ops))) { | |
1551 | printk(KERN_ERR "%s: rtnl_link_register failed\n", __func__); | |
1552 | return ret; | |
1553 | } | |
1554 | ||
1555 | #ifdef HOST | |
1556 | if ((ret = device_create_file(dev, &dev_attr_vnet))) { | |
1557 | printk(KERN_ERR "%s: device_create_file failed\n", __func__); | |
1558 | rtnl_link_unregister(&micvnet_link_ops); | |
1559 | return ret; | |
1560 | } | |
1561 | #endif | |
1562 | micvnet.created = 1; | |
1563 | return 0; | |
1564 | } | |
1565 | ||
1566 | void | |
1567 | micvnet_exit(void) | |
1568 | { | |
1569 | rtnl_link_unregister(&micvnet_link_ops); | |
1570 | } | |
1571 | ||
1572 | #ifndef HOST | |
1573 | static void __exit | |
1574 | _micvnet_module_exit(void) | |
1575 | { | |
1576 | mic_ctx_t *mic_ctx = &mic_ctx_g; | |
1577 | ||
1578 | micvnet_stop(mic_ctx); | |
1579 | micvnet_remove(mic_ctx); | |
1580 | micvnet_exit(); | |
1581 | } | |
1582 | ||
1583 | static int | |
1584 | micvnet_reboot(struct notifier_block *notifier, unsigned long unused1, void *unused2) | |
1585 | { | |
1586 | /* Calling _micvnet_module_exit() here will hang the uOS during shutdown in NFS | |
1587 | * root case */ | |
1588 | return NOTIFY_OK; | |
1589 | } | |
1590 | ||
1591 | static struct notifier_block micvnet_reboot_notifier = { | |
1592 | .notifier_call = micvnet_reboot, | |
1593 | .priority = 0, | |
1594 | }; | |
1595 | ||
1596 | void __exit | |
1597 | micvnet_module_exit(void) | |
1598 | { | |
1599 | unregister_reboot_notifier(&micvnet_reboot_notifier); | |
1600 | _micvnet_module_exit(); | |
1601 | } | |
1602 | ||
1603 | int __init | |
1604 | micvnet_module_init(void) | |
1605 | { | |
1606 | mic_ctx_t *mic_ctx = &mic_ctx_g; | |
1607 | int ret = 0; | |
1608 | ||
1609 | if ((ret = register_reboot_notifier(&micvnet_reboot_notifier))) { | |
1610 | printk(KERN_ERR "register_reboot_notifier failed: error %d\n", ret); | |
1611 | goto err_exit; | |
1612 | } | |
1613 | ||
1614 | memset(mic_ctx, 0, sizeof(*mic_ctx)); | |
1615 | mic_ctx->bi_id = 0; | |
1616 | ||
1617 | if ((ret = micvnet_init(NULL))) | |
1618 | goto err_exit_unregister_reboot_notifier; | |
1619 | if ((ret = micvnet_probe(mic_ctx))) | |
1620 | goto err_exit_micvnet_exit; | |
1621 | if ((ret = micvnet_start(mic_ctx))) | |
1622 | goto err_exit_micvnet_remove; | |
1623 | ||
1624 | return 0; | |
1625 | ||
1626 | err_exit_micvnet_remove: | |
1627 | micvnet_remove(mic_ctx); | |
1628 | err_exit_micvnet_exit: | |
1629 | micvnet_exit(); | |
1630 | err_exit_unregister_reboot_notifier: | |
1631 | unregister_reboot_notifier(&micvnet_reboot_notifier); | |
1632 | err_exit: | |
1633 | printk(KERN_ERR "%s failed: error %d\n", __func__, ret); | |
1634 | return ret; | |
1635 | } | |
1636 | ||
1637 | #ifdef STANDALONE_VNET_DMA | |
1638 | module_init(micvnet_module_init); | |
1639 | module_exit(micvnet_module_exit); | |
1640 | #endif | |
1641 | ||
1642 | MODULE_LICENSE("GPL"); | |
1643 | #endif |