Commit | Line | Data |
---|---|---|
800f879a AT |
1 | /* |
2 | * Copyright 2010-2017 Intel Corporation. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License, version 2, | |
6 | * as published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
11 | * General Public License for more details. | |
12 | * | |
13 | * Disclaimer: The codes contained in these modules may be specific to | |
14 | * the Intel Software Development Platform codenamed Knights Ferry, | |
15 | * and the Intel product codenamed Knights Corner, and are not backward | |
16 | * compatible with other Intel products. Additionally, Intel will NOT | |
17 | * support the codes or instruction set in future products. | |
18 | * | |
19 | * Intel offers no warranty of any kind regarding the code. This code is | |
20 | * licensed on an "AS IS" basis and Intel is not obligated to provide | |
21 | * any support, assistance, installation, training, or other services | |
22 | * of any kind. Intel is also not obligated to provide any updates, | |
23 | * enhancements or extensions. Intel specifically disclaims any warranty | |
24 | * of merchantability, non-infringement, fitness for any particular | |
25 | * purpose, and any other warranty. | |
26 | * | |
27 | * Further, Intel disclaims all liability of any kind, including but | |
28 | * not limited to liability for infringement of any proprietary rights, | |
29 | * relating to the use of the code, even if Intel is notified of the | |
30 | * possibility of such liability. Except as expressly stated in an Intel | |
31 | * license agreement provided with this code and agreed upon with Intel, | |
32 | * no license, express or implied, by estoppel or otherwise, to any | |
33 | * intellectual property rights is granted herein. | |
34 | */ | |
35 | ||
36 | #include "micint.h" | |
37 | #include "mic_common.h" | |
38 | #include <mic/micsboxdefine.h> | |
39 | #include <linux/ip.h> | |
40 | #include <linux/tcp.h> | |
41 | #include <linux/kernel.h> | |
42 | #include "mic/micveth.h" | |
43 | ||
44 | #define PWR_MGMT_NO_POLL_AFTER_LINKS_UP 1 | |
45 | ||
46 | /* | |
47 | In intr/poll modes, mic_smpt_uninit has already been called before | |
48 | micveth_destroy is called during rmmod. This results in host driver crash. The | |
49 | current workaround is, given the 'legacy' nature of VNET intr/poll modes, to | |
50 | not call mic_ctx_unmap_single() at rmmod. This workaround will result in some | |
51 | unmapped memory and a warn_on from micscif_smpt.c. | |
52 | */ | |
53 | #define WA_UNMAP_AT_RMMOD 0 | |
54 | ||
55 | static void micveth_clientpoll(struct work_struct *work); | |
56 | static void micveth_poll(struct work_struct *work); | |
57 | static int micvnet_host_doorbell_intr_handler(mic_ctx_t *mic_ctx, int doorbell); | |
58 | static void micvnet_intr_bh_handler(struct work_struct *work); | |
59 | void micveth_send_intr(micveth_info_t *veth_info); | |
60 | ||
61 | micveth_t micveth; | |
62 | ||
63 | void dump_skb(struct sk_buff *skb, int xmit); | |
64 | ||
65 | static inline | |
66 | mic_ctx_t *veth_to_ctx(micveth_info_t *veth_info) | |
67 | { | |
68 | return veth_info->mic_ctx; | |
69 | } | |
70 | ||
71 | static int | |
72 | micveth_set_address(struct net_device *dev, void *p) | |
73 | { | |
74 | struct sockaddr *sa = p; | |
75 | ||
76 | if (!is_valid_ether_addr(sa->sa_data)) | |
77 | return -EADDRNOTAVAIL; | |
78 | ||
79 | memcpy(dev->dev_addr, sa->sa_data, ETH_ALEN); | |
80 | return 0; | |
81 | } | |
82 | ||
83 | #if LINUX_VERSION_CODE < KERNEL_VERSION(3,2,0) | |
84 | static void | |
85 | micveth_multicast_list(struct net_device *dev) | |
86 | { | |
87 | } | |
88 | #endif | |
89 | ||
90 | static int | |
91 | micveth_deliver(struct sk_buff *skb, struct net_device *dev, micveth_info_t *veth_info) | |
92 | { | |
93 | veth_ring_t *ring; | |
94 | ring_queue_t *tx_queue; | |
95 | ring_desc_t *desc; | |
96 | ring_packet_t *packet; | |
97 | int next_tail; | |
98 | ||
99 | //dump_skb(skb, 1); | |
100 | ||
101 | spin_lock(&veth_info->vi_txlock); | |
102 | ring = &veth_info->vi_ring.ring; | |
103 | tx_queue = &ring->r_tx; | |
104 | ||
105 | next_tail = (tx_queue->rq_tail + 1) % tx_queue->rq_length; | |
106 | if (next_tail == tx_queue->rq_head) { | |
107 | // queue_full situation - just drop the packet and let the stack retry | |
108 | spin_unlock(&veth_info->vi_txlock); | |
109 | return 1; | |
110 | } | |
111 | ||
112 | desc = &tx_queue->rq_descs[tx_queue->rq_tail]; | |
113 | packet = &veth_info->vi_tx_desc[tx_queue->rq_tail]; | |
114 | packet->pd_skb = skb; | |
115 | packet->pd_phys = mic_ctx_map_single(veth_to_ctx(veth_info), | |
116 | skb->data, skb->len); | |
117 | packet->pd_length = skb->len; | |
118 | desc->rd_phys = packet->pd_phys; | |
119 | desc->rd_length = skb->len; | |
120 | desc->rd_valid = 1; | |
121 | ||
122 | /* | |
123 | * Need a write memory barrier between copying the skb data to | |
124 | * the buffer and updating the tail pointer. NOT an smp_wmb(), | |
125 | * because this memory barrier needs to be done even if there is | |
126 | * a single CPU in the system. | |
127 | */ | |
128 | wmb(); | |
129 | tx_queue->rq_tail = (tx_queue->rq_tail + 1) % tx_queue->rq_length; | |
130 | spin_unlock(&veth_info->vi_txlock); | |
131 | ||
132 | if (mic_vnet_mode == VNET_MODE_INTR) { | |
133 | micveth_send_intr(veth_info); | |
134 | } | |
135 | ||
136 | return 0; | |
137 | } | |
138 | ||
139 | static int | |
140 | micveth_xmit(struct sk_buff *skb, struct net_device *dev) | |
141 | { | |
142 | micveth_info_t *veth_info; | |
143 | ||
144 | if (be16_to_cpu(skb->protocol) == ETH_P_IPV6) { | |
145 | kfree_skb(skb); | |
146 | dev->stats.tx_dropped++; | |
147 | return NETDEV_TX_OK; | |
148 | } | |
149 | ||
150 | dev->stats.tx_packets++; | |
151 | dev->stats.tx_bytes += skb->len; | |
152 | ||
153 | veth_info = dev->ml_priv; | |
154 | ||
155 | if (veth_info->vi_state != VETH_STATE_LINKUP) { | |
156 | kfree_skb(skb); | |
157 | dev->stats.tx_dropped++; | |
158 | return NETDEV_TX_OK; | |
159 | } | |
160 | ||
161 | if (micveth_deliver(skb, dev, veth_info)) { | |
162 | kfree_skb(skb); | |
163 | dev->stats.tx_dropped++; | |
164 | } | |
165 | ||
166 | return NETDEV_TX_OK; | |
167 | } | |
168 | ||
169 | static int | |
170 | micveth_change_mtu(struct net_device *dev, int new_mtu) | |
171 | { | |
172 | dev->mtu = new_mtu; | |
173 | return 0; | |
174 | } | |
175 | ||
176 | /* Start callback */ | |
177 | static int | |
178 | micveth_start_dev(struct net_device *dev) | |
179 | { | |
180 | micveth_info_t *veth_info = dev->ml_priv; | |
181 | ||
182 | micveth_start(veth_info->mic_ctx); | |
183 | return 0; | |
184 | } | |
185 | ||
186 | /* Stop callback */ | |
187 | static int | |
188 | micveth_stop_dev(struct net_device *dev) | |
189 | { | |
190 | micveth_info_t *veth_info = dev->ml_priv; | |
191 | ||
192 | micveth_stop(veth_info->mic_ctx); | |
193 | return 0; | |
194 | } | |
195 | ||
196 | #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,28) | |
197 | static const struct net_device_ops veth_netdev_ops = { | |
198 | .ndo_open = micveth_start_dev, | |
199 | .ndo_stop = micveth_stop_dev, | |
200 | .ndo_start_xmit = micveth_xmit, | |
201 | .ndo_validate_addr = eth_validate_addr, | |
202 | #if LINUX_VERSION_CODE < KERNEL_VERSION(3,2,0) | |
203 | .ndo_set_multicast_list = micveth_multicast_list, | |
204 | #endif | |
205 | .ndo_set_mac_address = micveth_set_address, | |
206 | .ndo_change_mtu = micveth_change_mtu, | |
207 | }; | |
208 | #endif | |
209 | ||
210 | static void | |
211 | micveth_setup(struct net_device *dev) | |
212 | { | |
213 | #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,28) | |
214 | dev->hard_start_xmit = micveth_xmit; | |
215 | dev->set_multicast_list = micveth_multicast_list; | |
216 | dev->set_mac_address = micveth_set_address; | |
217 | #endif | |
218 | ether_setup(dev); | |
219 | ||
220 | /* Initialize the device structure. */ | |
221 | #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,28) | |
222 | dev->netdev_ops = &veth_netdev_ops; | |
223 | #endif | |
46724449 AT |
224 | dev->priv_destructor = free_netdev; |
225 | dev->needs_free_netdev = false; | |
800f879a AT |
226 | |
227 | /* Fill in device structure with ethernet-generic values. */ | |
228 | dev->mtu = (MICVETH_MAX_PACKET_SIZE); | |
229 | dev->tx_queue_len = 0; | |
230 | dev->flags &= ~IFF_MULTICAST; | |
231 | random_ether_addr(dev->dev_addr); | |
232 | } | |
233 | ||
234 | static int | |
46724449 | 235 | micveth_validate(struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *dummy) |
800f879a AT |
236 | { |
237 | if (tb[IFLA_ADDRESS]) { | |
238 | if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) | |
239 | return -EINVAL; | |
240 | if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) | |
241 | return -EADDRNOTAVAIL; | |
242 | } | |
243 | return 0; | |
244 | } | |
245 | ||
246 | static struct rtnl_link_ops micveth_link_ops __read_mostly = { | |
247 | .kind = "micveth", | |
248 | .setup = micveth_setup, | |
249 | .validate = micveth_validate, | |
250 | }; | |
251 | ||
252 | static int | |
253 | micveth_probe_int(micveth_info_t *veth_info, mic_ctx_t *mic_ctx) | |
254 | { | |
255 | struct net_device *dev_veth; | |
256 | ring_queue_t *queue; | |
257 | ring_desc_t *desc; | |
258 | ring_packet_t *packet; | |
259 | int idx; | |
260 | int err = 0; | |
261 | ||
262 | veth_info->vi_pdev = mic_ctx->bi_pdev; | |
263 | veth_info->vi_sbox = (uint8_t *)((unsigned long)mic_ctx->mmio.va + | |
264 | HOST_SBOX_BASE_ADDRESS); | |
265 | veth_info->vi_scratch14 = (uint32_t *)((unsigned long)mic_ctx->mmio.va + | |
266 | HOST_SBOX_BASE_ADDRESS + SBOX_SCRATCH14); | |
267 | veth_info->vi_scratch15 = (uint32_t *)((unsigned long)mic_ctx->mmio.va + | |
268 | HOST_SBOX_BASE_ADDRESS + SBOX_SCRATCH15); | |
269 | veth_info->mic_ctx = mic_ctx; | |
270 | mic_ctx->bi_vethinfo = (void *)veth_info; | |
271 | ||
272 | spin_lock_init(&veth_info->vi_txlock); | |
273 | spin_lock_init(&veth_info->vi_rxlock); | |
274 | ||
275 | if (mic_vnet_mode == VNET_MODE_POLL) | |
276 | INIT_DELAYED_WORK(&veth_info->vi_poll, micveth_poll); | |
277 | ||
278 | // Set the current sk_buff allocation size | |
279 | veth_info->vi_skb_mtu = MICVETH_MAX_PACKET_SIZE + 32; | |
280 | ||
281 | // Get the physical memory address for the ring descriptors | |
282 | veth_info->vi_ring.phys = mic_ctx_map_single(veth_to_ctx(veth_info), &veth_info->vi_ring.ring, | |
283 | sizeof(veth_ring_t)); | |
284 | veth_info->vi_ring.length = sizeof(veth_ring_t); | |
285 | ||
286 | queue = &veth_info->vi_ring.ring.r_tx; | |
287 | queue->rq_head = 0; | |
288 | queue->rq_tail = 0; | |
289 | queue->rq_length = MICVETH_TRANSFER_FIFO_SIZE; | |
290 | ||
291 | veth_info->vi_pend = 0; | |
292 | ||
293 | packet = &veth_info->vi_tx_desc[0]; | |
294 | for (idx = 0; idx < queue->rq_length; idx++) { | |
295 | desc = &queue->rq_descs[idx]; | |
296 | packet[idx].pd_skb = NULL; | |
297 | packet[idx].pd_phys = 0; | |
298 | packet[idx].pd_length = 0; | |
299 | ||
300 | desc->rd_phys = 0; | |
301 | desc->rd_length = 0; | |
302 | desc->rd_valid = 0; | |
303 | } | |
304 | ||
305 | // This is the recieve end. | |
306 | queue = &veth_info->vi_ring.ring.r_rx; | |
307 | queue->rq_head = 0; | |
308 | queue->rq_tail = 0; | |
309 | queue->rq_length = MICVETH_TRANSFER_FIFO_SIZE; | |
310 | ||
311 | packet = &veth_info->vi_rx_desc[0]; | |
312 | for (idx = 0; idx < queue->rq_length; idx++) { | |
313 | desc = &queue->rq_descs[idx]; | |
314 | if (!(packet[idx].pd_skb = dev_alloc_skb(veth_info->vi_skb_mtu))) | |
315 | return -ENOMEM; | |
316 | packet[idx].pd_phys = mic_ctx_map_single(veth_to_ctx(veth_info), packet[idx].pd_skb->data, | |
317 | veth_info->vi_skb_mtu); | |
318 | packet[idx].pd_length = veth_info->vi_skb_mtu; | |
319 | ||
320 | desc->rd_phys = packet[idx].pd_phys; | |
321 | desc->rd_length = packet[idx].pd_length; | |
322 | desc->rd_valid = 1; | |
323 | } | |
324 | #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) | |
325 | if ((dev_veth = alloc_netdev(sizeof(micveth_info_t), "mic%d", micveth_setup)) == NULL) { | |
326 | #else | |
327 | if ((dev_veth = alloc_netdev(sizeof(micveth_info_t), "mic%d", NET_NAME_UNKNOWN, micveth_setup)) == NULL) { | |
328 | #endif | |
329 | return -ENOMEM; | |
330 | } | |
331 | ||
332 | veth_info->vi_netdev = dev_veth; | |
333 | dev_veth->ml_priv = veth_info; | |
334 | dev_veth->rtnl_link_ops = &micveth_link_ops; | |
335 | ||
336 | if ((err = register_netdev(dev_veth)) < 0) { | |
337 | printk("register netdev failed %d\n", err); | |
338 | free_netdev(dev_veth); | |
339 | return err; | |
340 | } | |
341 | ||
342 | veth_info->vi_state = VETH_STATE_INITIALIZED; | |
343 | return 0; | |
344 | } | |
345 | ||
346 | static ssize_t show_veth(struct device *dev, | |
347 | struct device_attribute *attr, char *buf); | |
348 | DEVICE_ATTR(veth, S_IRUGO, show_veth, NULL); | |
349 | ||
350 | static int | |
351 | micveth_init_int(int num_bds, struct device *dev) | |
352 | { | |
353 | int bd; | |
354 | int err = 0; | |
355 | ||
356 | micveth.lv_num_interfaces = num_bds; | |
357 | micveth.lv_num_clients = num_bds; | |
358 | micveth.lv_active_clients = 0; | |
359 | micveth.lv_num_links_remaining = num_bds; | |
360 | ||
361 | BUG_ON(rtnl_link_register(&micveth_link_ops)); | |
362 | ||
363 | // Allocate space for the control of each device in the system. | |
364 | micveth.lv_info = kmalloc(sizeof(micveth_info_t) * num_bds, GFP_KERNEL); | |
365 | ||
366 | // Initialize state mutex. Overloaded use for several fields. | |
367 | mutex_init(&micveth.lv_state_mutex); | |
368 | ||
369 | // Setup of timer for probeing active mic clients. When the total active board | |
370 | // count is zero the poll is not running. | |
371 | micveth.lv_pollstate = CLIENT_POLL_STOPPED; | |
372 | INIT_DELAYED_WORK(&micveth.lv_poll, micveth_clientpoll); | |
373 | init_waitqueue_head(&micveth.lv_wq); | |
374 | ||
375 | // Init each of the existing boards. | |
376 | for (bd = 0; bd < num_bds; bd++) { | |
377 | micveth_probe_int(&micveth.lv_info[bd], &mic_data.dd_bi[bd]->bi_ctx); | |
378 | } | |
379 | ||
380 | err = device_create_file(dev, &dev_attr_veth); | |
381 | return err; | |
382 | } | |
383 | ||
384 | static void | |
385 | micveth_exit_int(void) | |
386 | { | |
387 | mic_ctx_t *mic_ctx = kmalloc(sizeof(mic_ctx_t), GFP_KERNEL); | |
388 | micveth_info_t *veth_info; | |
389 | ring_packet_t *packet; | |
390 | int bd; | |
391 | int idx; | |
392 | ||
393 | rtnl_link_unregister(&micveth_link_ops); | |
394 | ||
395 | for (bd = 0; bd < micveth.lv_num_clients; bd++) { | |
396 | veth_info = &micveth.lv_info[bd]; | |
397 | ||
398 | /* veth_info->mic_ctx == mic_data.dd_bi[bd] is freed in | |
399 | remove so cannot be used in exit */ | |
400 | mic_ctx->bi_vethinfo = veth_info; | |
401 | micveth_stop(mic_ctx); | |
402 | ||
403 | #if WA_UNMAP_AT_RMMOD | |
404 | mic_ctx_unmap_single(veth_to_ctx(veth_info), veth_info->vi_ring.phys, | |
405 | sizeof(veth_ring_t)); | |
406 | #endif | |
407 | ||
408 | for (idx = 0; idx < veth_info->vi_ring.ring.r_tx.rq_length; idx++) { | |
409 | packet = &veth_info->vi_tx_desc[idx]; | |
410 | if (packet->pd_skb != NULL) { | |
411 | #if WA_UNMAP_AT_RMMOD | |
412 | mic_ctx_unmap_single(veth_to_ctx(veth_info), packet->pd_phys, | |
413 | packet->pd_skb->len); | |
414 | #endif | |
415 | kfree_skb(packet->pd_skb); | |
416 | } | |
417 | } | |
418 | ||
419 | for (idx = 0; idx < veth_info->vi_ring.ring.r_rx.rq_length; idx++) { | |
420 | packet = &veth_info->vi_rx_desc[idx]; | |
421 | #if WA_UNMAP_AT_RMMOD | |
422 | mic_ctx_unmap_single(veth_to_ctx(veth_info), packet->pd_phys, packet->pd_skb->len); | |
423 | #endif | |
424 | kfree_skb(packet->pd_skb); | |
425 | } | |
426 | } | |
427 | ||
428 | kfree(mic_ctx); | |
429 | kfree(micveth.lv_info); | |
430 | } | |
431 | ||
432 | static int | |
433 | micveth_start_int(mic_ctx_t *mic_ctx) | |
434 | { | |
435 | micveth_info_t *veth_info = &micveth.lv_info[mic_ctx->bi_id]; | |
436 | ||
437 | // Eventuall (very soon) most of the descriptor allocation for a board will be done here | |
438 | if (veth_info->vi_state != VETH_STATE_INITIALIZED) | |
439 | return 0; | |
440 | ||
441 | mutex_lock(&micveth.lv_state_mutex); | |
442 | ||
443 | if (micveth.lv_pollstate == CLIENT_POLL_STOPPED) { | |
444 | schedule_delayed_work(&micveth.lv_poll, msecs_to_jiffies(MICVETH_CLIENT_TIMER_DELAY)); | |
445 | micveth.lv_pollstate = CLIENT_POLL_RUNNING; | |
446 | } | |
447 | ||
448 | micveth.lv_active_clients++; | |
449 | mutex_unlock(&micveth.lv_state_mutex); | |
450 | ||
451 | veth_info->vi_pend = 0; | |
452 | ||
453 | veth_info->vi_ring.ring.r_tx.rq_head = 0; | |
454 | veth_info->vi_ring.ring.r_tx.rq_tail = 0; | |
455 | ||
456 | veth_info->vi_ring.ring.r_rx.rq_head = 0; | |
457 | veth_info->vi_ring.ring.r_rx.rq_tail = 0; | |
458 | veth_info->vi_state = VETH_STATE_LINKDOWN; | |
459 | ||
460 | if (mic_vnet_mode == VNET_MODE_INTR) { | |
461 | snprintf(veth_info->vi_wqname, sizeof(veth_info->vi_wqname), | |
462 | "VNET INTR %d\n", mic_ctx->bi_id); | |
463 | veth_info->vi_wq = create_singlethread_workqueue(veth_info->vi_wqname); | |
464 | INIT_WORK(&veth_info->vi_bh, micvnet_intr_bh_handler); | |
465 | ||
466 | // Install interrupt handler on doorbell 3 | |
467 | mic_reg_irqhandler(mic_ctx, 3, "Host DoorBell 3", | |
468 | micvnet_host_doorbell_intr_handler); | |
469 | } | |
470 | ||
471 | return 0; | |
472 | } | |
473 | ||
474 | static void | |
475 | micveth_stop_int(mic_ctx_t *mic_ctx) | |
476 | { | |
477 | micveth_info_t *veth_info = (micveth_info_t *)(mic_ctx->bi_vethinfo); | |
478 | ||
479 | if (veth_info->vi_state == VETH_STATE_INITIALIZED) | |
480 | return; | |
481 | ||
482 | mutex_lock(&micveth.lv_state_mutex); | |
483 | ||
484 | if (mic_vnet_mode == VNET_MODE_INTR) { | |
485 | // Remove interrupt handler on doorbell 3 | |
486 | mic_unreg_irqhandler(mic_ctx, 3, "Host DoorBell 3"); | |
487 | ||
488 | destroy_workqueue(veth_info->vi_wq); | |
489 | } | |
490 | ||
491 | micveth.lv_active_clients--; | |
492 | veth_info->vi_state = VETH_STATE_INITIALIZED; | |
493 | ||
494 | if (micveth.lv_active_clients) { | |
495 | mutex_unlock(&micveth.lv_state_mutex); | |
496 | return; | |
497 | } | |
498 | ||
499 | micveth.lv_num_links_remaining = micveth.lv_num_clients; | |
500 | ||
501 | #if PWR_MGMT_NO_POLL_AFTER_LINKS_UP | |
502 | micveth.lv_pollstate = CLIENT_POLL_STOPPED; | |
503 | mutex_unlock(&micveth.lv_state_mutex); | |
504 | #else | |
505 | micveth.lv_pollstate = CLIENT_POLL_STOPPING; | |
506 | mutex_unlock(&micveth.lv_state_mutex); | |
507 | wait_event(micveth.lv_wq, micveth.lv_pollstate == CLIENT_POLL_STOPPED); | |
508 | #endif | |
509 | } | |
510 | ||
511 | #define NO_SRATCHREGREAD_AFTER_CONNECT 1 | |
512 | static void | |
513 | micveth_clientpoll(struct work_struct *work) | |
514 | { | |
515 | micveth_info_t *veth_info; | |
516 | uint32_t transRingHi; | |
517 | uint32_t transRingLo; | |
518 | uint32_t scratch14 = 0; | |
519 | uint32_t scratch15 = 0; | |
520 | int bd; | |
521 | static int enter = 0; | |
522 | ||
523 | if (enter == 0) | |
524 | { | |
525 | printk("micveth is polling\n"); | |
526 | enter = 1; | |
527 | } | |
528 | ||
529 | mutex_lock(&micveth.lv_state_mutex); | |
530 | if (micveth.lv_pollstate == CLIENT_POLL_STOPPING) { | |
531 | micveth.lv_pollstate = CLIENT_POLL_STOPPED; | |
532 | mutex_unlock(&micveth.lv_state_mutex); | |
533 | wake_up(&micveth.lv_wq); | |
534 | return; | |
535 | } | |
536 | ||
537 | // Check for state changes for each board in the system | |
538 | for (bd = 0; bd < micveth.lv_num_clients; bd++) { | |
539 | veth_info = &micveth.lv_info[bd]; | |
540 | ||
541 | // Do not poll boards that have not had the interface started. | |
542 | if (veth_info->vi_state == VETH_STATE_INITIALIZED) { | |
543 | break; | |
544 | } | |
545 | ||
546 | #ifdef NO_SRATCHREGREAD_AFTER_CONNECT | |
547 | if(veth_info->vi_state != VETH_STATE_LINKUP) { | |
548 | #endif | |
549 | scratch14 = readl(veth_info->vi_scratch14); | |
550 | scratch15 = readl(veth_info->vi_scratch15); | |
551 | #ifdef NO_SRATCHREGREAD_AFTER_CONNECT | |
552 | } | |
553 | #endif | |
554 | ||
555 | if (veth_info->vi_state == VETH_STATE_LINKUP) { | |
556 | if (scratch14 == MICVETH_LINK_DOWN_MAGIC) { | |
557 | veth_info->vi_state = VETH_STATE_LINKDOWN; | |
558 | } | |
559 | } else if (veth_info->vi_state == VETH_STATE_LINKDOWN) { | |
560 | if (scratch14 == MICVETH_LINK_UP_MAGIC) { | |
561 | // Write the transfer ring address. | |
562 | transRingHi = (uint32_t)(veth_info->vi_ring.phys >> 32); | |
563 | transRingLo = (uint32_t)(veth_info->vi_ring.phys & 0xffffffff); | |
564 | ||
565 | writel(transRingLo, veth_info->vi_scratch14); | |
566 | writel(transRingHi, veth_info->vi_scratch15); | |
567 | ||
568 | veth_info->vi_state = VETH_STATE_LINKUP; | |
569 | printk("MIC virtual ethernet up for board %d\n", bd); | |
570 | #ifdef MIC_IS_EMULATION | |
571 | printk("Card wrote Magic: It must be UP!\n"); | |
572 | #endif | |
573 | ||
574 | if (mic_vnet_mode == VNET_MODE_POLL) { | |
575 | schedule_delayed_work(&veth_info->vi_poll, | |
576 | msecs_to_jiffies(MICVETH_POLL_TIMER_DELAY)); | |
577 | } | |
578 | ||
579 | micveth.lv_num_links_remaining--; | |
580 | } | |
581 | #ifdef MIC_IS_EMULATION | |
582 | else if (scratch14) { | |
583 | printk("---> 0x%x \n", scratch14); | |
584 | writel(0x0, veth_info->vi_scratch14); | |
585 | } | |
586 | #endif | |
587 | } | |
588 | } | |
589 | ||
590 | mutex_unlock(&micveth.lv_state_mutex); | |
591 | ||
592 | #if PWR_MGMT_NO_POLL_AFTER_LINKS_UP | |
593 | if (micveth.lv_num_links_remaining) | |
594 | #endif | |
595 | schedule_delayed_work(&micveth.lv_poll, msecs_to_jiffies(MICVETH_CLIENT_TIMER_DELAY)); | |
596 | } | |
597 | ||
598 | static int | |
599 | micvnet_host_doorbell_intr_handler(mic_ctx_t *mic_ctx, int doorbell) | |
600 | { | |
601 | micveth_info_t *veth_info; | |
602 | veth_info = &micveth.lv_info[mic_ctx->bi_id]; | |
603 | queue_work(veth_info->vi_wq, &veth_info->vi_bh); | |
604 | return 0; | |
605 | } | |
606 | ||
607 | void | |
608 | micveth_send_intr(micveth_info_t *veth_info) | |
609 | { | |
610 | mic_ctx_t *mic_ctx = veth_info->mic_ctx; | |
611 | mic_send_vnet_intr(mic_ctx); | |
612 | } | |
613 | ||
614 | void | |
615 | _micveth_process_descriptors(micveth_info_t *veth_info) | |
616 | { | |
617 | veth_ring_t *ring = &veth_info->vi_ring.ring; | |
618 | ring_queue_t *rx_queue = &ring->r_rx; | |
619 | ring_queue_t *tx_queue = &ring->r_tx; | |
620 | ring_desc_t *desc; | |
621 | ring_packet_t *packet; | |
622 | struct sk_buff *skb; | |
623 | int receive_skb = 0; | |
624 | int err; | |
625 | ||
626 | if (veth_info->vi_state != VETH_STATE_LINKUP) { | |
627 | return; | |
628 | } | |
629 | ||
630 | spin_lock_bh(&veth_info->vi_rxlock); | |
631 | ||
632 | while (rx_queue->rq_head != rx_queue->rq_tail) { | |
633 | desc = &rx_queue->rq_descs[rx_queue->rq_head]; | |
634 | ||
635 | veth_info->vi_netdev->stats.rx_packets++; | |
636 | veth_info->vi_netdev->stats.rx_bytes += desc->rd_length; | |
637 | ||
638 | packet = &veth_info->vi_rx_desc[rx_queue->rq_head]; | |
639 | ||
640 | skb = packet->pd_skb; | |
641 | skb_put(skb, desc->rd_length); | |
642 | ||
643 | //dump_skb(skb, 0); | |
644 | mic_ctx_unmap_single(veth_to_ctx(veth_info), packet->pd_phys, veth_info->vi_skb_mtu); | |
645 | packet->pd_skb = dev_alloc_skb(veth_info->vi_skb_mtu); | |
646 | packet->pd_phys = mic_ctx_map_single(veth_to_ctx(veth_info), packet->pd_skb->data, | |
647 | veth_info->vi_skb_mtu); | |
648 | desc->rd_phys = packet->pd_phys; | |
649 | desc->rd_length = packet->pd_length; | |
650 | ||
651 | skb->dev = veth_info->vi_netdev; | |
652 | skb->protocol = eth_type_trans(skb, skb->dev); | |
653 | skb->ip_summed = CHECKSUM_NONE; | |
654 | ||
655 | err = netif_receive_skb(skb); | |
656 | /* | |
657 | * Need a general memory barrier between copying the data from | |
658 | * the buffer and updating the head pointer. It's the general | |
659 | * mb() because we're ordering the read of the data with the write. | |
660 | */ | |
661 | mb(); | |
662 | rx_queue->rq_head = (rx_queue->rq_head + 1) % rx_queue->rq_length; | |
663 | receive_skb++; | |
664 | } | |
665 | ||
666 | /* Send intr to TX so that pending SKB's can be freed */ | |
667 | if (receive_skb && mic_vnet_mode == VNET_MODE_INTR) { | |
668 | micveth_send_intr(veth_info); | |
669 | } | |
670 | ||
671 | spin_unlock_bh(&veth_info->vi_rxlock); | |
672 | ||
673 | spin_lock_bh(&veth_info->vi_txlock); | |
674 | ||
675 | // Also handle completed tx requests | |
676 | while (veth_info->vi_pend != tx_queue->rq_head) { | |
677 | desc = &tx_queue->rq_descs[veth_info->vi_pend]; | |
678 | packet = &veth_info->vi_tx_desc[veth_info->vi_pend]; | |
679 | ||
680 | skb = packet->pd_skb; | |
681 | packet->pd_skb = NULL; | |
682 | ||
683 | mic_ctx_unmap_single(veth_to_ctx(veth_info), packet->pd_phys, skb->len); | |
684 | packet->pd_phys = 0; | |
685 | ||
686 | kfree_skb(skb); | |
687 | ||
688 | veth_info->vi_pend = (veth_info->vi_pend + 1) % tx_queue->rq_length; | |
689 | } | |
690 | ||
691 | spin_unlock_bh(&veth_info->vi_txlock); | |
692 | ||
693 | if (mic_vnet_mode == VNET_MODE_POLL) { | |
694 | schedule_delayed_work(&veth_info->vi_poll, msecs_to_jiffies(MICVETH_POLL_TIMER_DELAY)); | |
695 | } | |
696 | } | |
697 | ||
698 | static void | |
699 | micvnet_intr_bh_handler(struct work_struct *work) | |
700 | { | |
701 | micveth_info_t *veth_info = container_of(work, micveth_info_t, vi_bh); | |
702 | _micveth_process_descriptors(veth_info); | |
703 | } | |
704 | ||
705 | static void | |
706 | micveth_poll(struct work_struct *work) | |
707 | { | |
708 | micveth_info_t *veth_info = container_of(work, micveth_info_t, vi_poll.work); | |
709 | ||
710 | _micveth_process_descriptors(veth_info); | |
711 | } | |
712 | ||
713 | static ssize_t | |
714 | show_veth(struct device *dev, struct device_attribute *attr, char *buf) | |
715 | { | |
716 | return snprintf(buf, PAGE_SIZE, "%s\n", | |
717 | micveth.lv_pollstate == CLIENT_POLL_RUNNING ? | |
718 | "running" : "stopped"); | |
719 | } | |
720 | ||
721 | /* | |
722 | VNET driver public API. These are simply wrappers which either invoke the old | |
723 | interrupt/poll mode functions or the new DMA mode functions. These are temporary and | |
724 | will be phased out with the old interrupt/poll mode so only the DMA mode will be around | |
725 | eventually. | |
726 | */ | |
727 | int __init | |
728 | micveth_init(struct device *dev) | |
729 | { | |
730 | printk("vnet: mode: %s, buffers: %d\n", | |
731 | mic_vnet_modes[mic_vnet_mode], vnet_num_buffers); | |
732 | ||
733 | if (mic_vnet_mode == VNET_MODE_DMA) | |
734 | return micvnet_init(dev); | |
735 | /* Intr/poll modes use micveth_init_legacy */ | |
736 | return 0; | |
737 | } | |
738 | ||
739 | int __init | |
740 | micveth_init_legacy(int num_bds, struct device *dev) | |
741 | { | |
742 | if (mic_vnet_mode != VNET_MODE_DMA) | |
743 | return micveth_init_int(num_bds, dev); | |
744 | /* DMA mode uses micveth_init */ | |
745 | return 0; | |
746 | } | |
747 | ||
748 | void | |
749 | micveth_exit(void) | |
750 | { | |
751 | if (mic_vnet_mode == VNET_MODE_DMA) | |
752 | micvnet_exit(); | |
753 | else | |
754 | micveth_exit_int(); | |
755 | } | |
756 | ||
757 | int | |
758 | micveth_probe(mic_ctx_t *mic_ctx) | |
759 | { | |
760 | if (mic_vnet_mode == VNET_MODE_DMA) | |
761 | return micvnet_probe(mic_ctx); | |
762 | /* No support for micveth_probe in legacy intr/poll modes */ | |
763 | return 0; | |
764 | } | |
765 | ||
766 | void | |
767 | micveth_remove(mic_ctx_t *mic_ctx) | |
768 | { | |
769 | if (mic_vnet_mode == VNET_MODE_DMA) | |
770 | micvnet_remove(mic_ctx); | |
771 | /* No support for micveth_remove in legacy intr/poll modes */ | |
772 | } | |
773 | ||
774 | int | |
775 | micveth_start(mic_ctx_t *mic_ctx) | |
776 | { | |
777 | micveth_info_t *veth_info = mic_ctx->bi_vethinfo; | |
778 | int err; | |
779 | ||
780 | if (mic_vnet_mode == VNET_MODE_DMA) | |
781 | err = micvnet_start(mic_ctx); | |
782 | else | |
783 | err = micveth_start_int(mic_ctx); | |
784 | ||
785 | if (!err) | |
786 | netif_carrier_on(veth_info->vi_netdev); | |
787 | ||
788 | return err; | |
789 | } | |
790 | ||
791 | void | |
792 | micveth_stop(mic_ctx_t *mic_ctx) | |
793 | { | |
794 | micveth_info_t *veth_info = mic_ctx->bi_vethinfo; | |
795 | ||
796 | if (mic_vnet_mode == VNET_MODE_DMA) | |
797 | micvnet_stop(mic_ctx); | |
798 | else | |
799 | micveth_stop_int(mic_ctx); | |
800 | ||
801 | if (veth_info) | |
802 | netif_carrier_off(veth_info->vi_netdev); | |
803 | } |