1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
4 #include "i40evf.h"
5 #include "i40e_prototype.h"
6 #include "i40evf_client.h"
7 /* All i40evf tracepoints are defined by the include below, which must
8 * be included exactly once across the whole kernel with
9 * CREATE_TRACE_POINTS defined
10 */
11 #define CREATE_TRACE_POINTS
12 #include "i40e_trace.h"
14 static int i40evf_setup_all_tx_resources(struct i40evf_adapter *adapter);
15 static int i40evf_setup_all_rx_resources(struct i40evf_adapter *adapter);
16 static int i40evf_close(struct net_device *netdev);
18 char i40evf_driver_name[] = "i40evf";
19 static const char i40evf_driver_string[] =
20 "Intel(R) 40-10 Gigabit Virtual Function Network Driver";
22 #define DRV_KERN "-k"
24 #define DRV_VERSION_MAJOR 3
25 #define DRV_VERSION_MINOR 2
26 #define DRV_VERSION_BUILD 2
27 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
28 __stringify(DRV_VERSION_MINOR) "." \
29 __stringify(DRV_VERSION_BUILD) \
30 DRV_KERN
31 const char i40evf_driver_version[] = DRV_VERSION;
32 static const char i40evf_copyright[] =
33 "Copyright (c) 2013 - 2015 Intel Corporation.";
35 /* i40evf_pci_tbl - PCI Device ID Table
36 *
37 * Wildcard entries (PCI_ANY_ID) should come last
38 * Last entry must be all 0s
39 *
40 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
41 * Class, Class Mask, private data (not used) }
42 */
43 static const struct pci_device_id i40evf_pci_tbl[] = {
44 {PCI_VDEVICE(INTEL, I40E_DEV_ID_VF), 0},
45 {PCI_VDEVICE(INTEL, I40E_DEV_ID_VF_HV), 0},
46 {PCI_VDEVICE(INTEL, I40E_DEV_ID_X722_VF), 0},
47 {PCI_VDEVICE(INTEL, I40E_DEV_ID_ADAPTIVE_VF), 0},
48 /* required last entry */
49 {0, }
50 };
52 MODULE_DEVICE_TABLE(pci, i40evf_pci_tbl);
54 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
55 MODULE_DESCRIPTION("Intel(R) XL710 X710 Virtual Function Network Driver");
56 MODULE_LICENSE("GPL");
57 MODULE_VERSION(DRV_VERSION);
59 static struct workqueue_struct *i40evf_wq;
61 /**
62 * i40evf_allocate_dma_mem_d - OS specific memory alloc for shared code
63 * @hw: pointer to the HW structure
64 * @mem: ptr to mem struct to fill out
65 * @size: size of memory requested
66 * @alignment: what to align the allocation to
67 **/
68 i40e_status i40evf_allocate_dma_mem_d(struct i40e_hw *hw,
69 struct i40e_dma_mem *mem,
70 u64 size, u32 alignment)
71 {
72 struct i40evf_adapter *adapter = (struct i40evf_adapter *)hw->back;
74 if (!mem)
75 return I40E_ERR_PARAM;
77 mem->size = ALIGN(size, alignment);
78 mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size,
79 (dma_addr_t *)&mem->pa, GFP_KERNEL);
80 if (mem->va)
81 return 0;
82 else
83 return I40E_ERR_NO_MEMORY;
84 }
86 /**
87 * i40evf_free_dma_mem_d - OS specific memory free for shared code
88 * @hw: pointer to the HW structure
89 * @mem: ptr to mem struct to free
90 **/
91 i40e_status i40evf_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
92 {
93 struct i40evf_adapter *adapter = (struct i40evf_adapter *)hw->back;
95 if (!mem || !mem->va)
96 return I40E_ERR_PARAM;
97 dma_free_coherent(&adapter->pdev->dev, mem->size,
98 mem->va, (dma_addr_t)mem->pa);
99 return 0;
100 }
102 /**
103 * i40evf_allocate_virt_mem_d - OS specific memory alloc for shared code
104 * @hw: pointer to the HW structure
105 * @mem: ptr to mem struct to fill out
106 * @size: size of memory requested
107 **/
108 i40e_status i40evf_allocate_virt_mem_d(struct i40e_hw *hw,
109 struct i40e_virt_mem *mem, u32 size)
110 {
111 if (!mem)
112 return I40E_ERR_PARAM;
114 mem->size = size;
115 mem->va = kzalloc(size, GFP_KERNEL);
117 if (mem->va)
118 return 0;
119 else
120 return I40E_ERR_NO_MEMORY;
121 }
123 /**
124 * i40evf_free_virt_mem_d - OS specific memory free for shared code
125 * @hw: pointer to the HW structure
126 * @mem: ptr to mem struct to free
127 **/
128 i40e_status i40evf_free_virt_mem_d(struct i40e_hw *hw,
129 struct i40e_virt_mem *mem)
130 {
131 if (!mem)
132 return I40E_ERR_PARAM;
134 /* it's ok to kfree a NULL pointer */
135 kfree(mem->va);
137 return 0;
138 }
140 /**
141 * i40evf_debug_d - OS dependent version of debug printing
142 * @hw: pointer to the HW structure
143 * @mask: debug level mask
144 * @fmt_str: printf-type format description
145 **/
146 void i40evf_debug_d(void *hw, u32 mask, char *fmt_str, ...)
147 {
148 char buf[512];
149 va_list argptr;
151 if (!(mask & ((struct i40e_hw *)hw)->debug_mask))
152 return;
154 va_start(argptr, fmt_str);
155 vsnprintf(buf, sizeof(buf), fmt_str, argptr);
156 va_end(argptr);
158 /* the debug string is already formatted with a newline */
159 pr_info("%s", buf);
160 }
162 /**
163 * i40evf_schedule_reset - Set the flags and schedule a reset event
164 * @adapter: board private structure
165 **/
166 void i40evf_schedule_reset(struct i40evf_adapter *adapter)
167 {
168 if (!(adapter->flags &
169 (I40EVF_FLAG_RESET_PENDING | I40EVF_FLAG_RESET_NEEDED))) {
170 adapter->flags |= I40EVF_FLAG_RESET_NEEDED;
171 schedule_work(&adapter->reset_task);
172 }
173 }
175 /**
176 * i40evf_tx_timeout - Respond to a Tx Hang
177 * @netdev: network interface device structure
178 **/
179 static void i40evf_tx_timeout(struct net_device *netdev)
180 {
181 struct i40evf_adapter *adapter = netdev_priv(netdev);
183 adapter->tx_timeout_count++;
184 i40evf_schedule_reset(adapter);
185 }
187 /**
188 * i40evf_misc_irq_disable - Mask off interrupt generation on the NIC
189 * @adapter: board private structure
190 **/
191 static void i40evf_misc_irq_disable(struct i40evf_adapter *adapter)
192 {
193 struct i40e_hw *hw = &adapter->hw;
195 if (!adapter->msix_entries)
196 return;
198 wr32(hw, I40E_VFINT_DYN_CTL01, 0);
200 /* read flush */
201 rd32(hw, I40E_VFGEN_RSTAT);
203 synchronize_irq(adapter->msix_entries[0].vector);
204 }
206 /**
207 * i40evf_misc_irq_enable - Enable default interrupt generation settings
208 * @adapter: board private structure
209 **/
210 static void i40evf_misc_irq_enable(struct i40evf_adapter *adapter)
211 {
212 struct i40e_hw *hw = &adapter->hw;
214 wr32(hw, I40E_VFINT_DYN_CTL01, I40E_VFINT_DYN_CTL01_INTENA_MASK |
215 I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
216 wr32(hw, I40E_VFINT_ICR0_ENA1, I40E_VFINT_ICR0_ENA1_ADMINQ_MASK);
218 /* read flush */
219 rd32(hw, I40E_VFGEN_RSTAT);
220 }
222 /**
223 * i40evf_irq_disable - Mask off interrupt generation on the NIC
224 * @adapter: board private structure
225 **/
226 static void i40evf_irq_disable(struct i40evf_adapter *adapter)
227 {
228 int i;
229 struct i40e_hw *hw = &adapter->hw;
231 if (!adapter->msix_entries)
232 return;
234 for (i = 1; i < adapter->num_msix_vectors; i++) {
235 wr32(hw, I40E_VFINT_DYN_CTLN1(i - 1), 0);
236 synchronize_irq(adapter->msix_entries[i].vector);
237 }
238 /* read flush */
239 rd32(hw, I40E_VFGEN_RSTAT);
240 }
242 /**
243 * i40evf_irq_enable_queues - Enable interrupt for specified queues
244 * @adapter: board private structure
245 * @mask: bitmap of queues to enable
246 **/
247 void i40evf_irq_enable_queues(struct i40evf_adapter *adapter, u32 mask)
248 {
249 struct i40e_hw *hw = &adapter->hw;
250 int i;
252 for (i = 1; i < adapter->num_msix_vectors; i++) {
253 if (mask & BIT(i - 1)) {
254 wr32(hw, I40E_VFINT_DYN_CTLN1(i - 1),
255 I40E_VFINT_DYN_CTLN1_INTENA_MASK |
256 I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK);
257 }
258 }
259 }
261 /**
262 * i40evf_irq_enable - Enable default interrupt generation settings
263 * @adapter: board private structure
264 * @flush: boolean value whether to run rd32()
265 **/
266 void i40evf_irq_enable(struct i40evf_adapter *adapter, bool flush)
267 {
268 struct i40e_hw *hw = &adapter->hw;
270 i40evf_misc_irq_enable(adapter);
271 i40evf_irq_enable_queues(adapter, ~0);
273 if (flush)
274 rd32(hw, I40E_VFGEN_RSTAT);
275 }
277 /**
278 * i40evf_msix_aq - Interrupt handler for vector 0
279 * @irq: interrupt number
280 * @data: pointer to netdev
281 **/
282 static irqreturn_t i40evf_msix_aq(int irq, void *data)
283 {
284 struct net_device *netdev = data;
285 struct i40evf_adapter *adapter = netdev_priv(netdev);
286 struct i40e_hw *hw = &adapter->hw;
288 /* handle non-queue interrupts, these reads clear the registers */
289 rd32(hw, I40E_VFINT_ICR01);
290 rd32(hw, I40E_VFINT_ICR0_ENA1);
292 /* schedule work on the private workqueue */
293 schedule_work(&adapter->adminq_task);
295 return IRQ_HANDLED;
296 }
298 /**
299 * i40evf_msix_clean_rings - MSIX mode Interrupt Handler
300 * @irq: interrupt number
301 * @data: pointer to a q_vector
302 **/
303 static irqreturn_t i40evf_msix_clean_rings(int irq, void *data)
304 {
305 struct i40e_q_vector *q_vector = data;
307 if (!q_vector->tx.ring && !q_vector->rx.ring)
308 return IRQ_HANDLED;
310 napi_schedule_irqoff(&q_vector->napi);
312 return IRQ_HANDLED;
313 }
315 /**
316 * i40evf_map_vector_to_rxq - associate irqs with rx queues
317 * @adapter: board private structure
318 * @v_idx: interrupt number
319 * @r_idx: queue number
320 **/
321 static void
322 i40evf_map_vector_to_rxq(struct i40evf_adapter *adapter, int v_idx, int r_idx)
323 {
324 struct i40e_q_vector *q_vector = &adapter->q_vectors[v_idx];
325 struct i40e_ring *rx_ring = &adapter->rx_rings[r_idx];
326 struct i40e_hw *hw = &adapter->hw;
328 rx_ring->q_vector = q_vector;
329 rx_ring->next = q_vector->rx.ring;
330 rx_ring->vsi = &adapter->vsi;
331 q_vector->rx.ring = rx_ring;
332 q_vector->rx.count++;
333 q_vector->rx.next_update = jiffies + 1;
334 q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
335 q_vector->ring_mask |= BIT(r_idx);
336 wr32(hw, I40E_VFINT_ITRN1(I40E_RX_ITR, q_vector->reg_idx),
337 q_vector->rx.current_itr);
338 q_vector->rx.current_itr = q_vector->rx.target_itr;
339 }
341 /**
342 * i40evf_map_vector_to_txq - associate irqs with tx queues
343 * @adapter: board private structure
344 * @v_idx: interrupt number
345 * @t_idx: queue number
346 **/
347 static void
348 i40evf_map_vector_to_txq(struct i40evf_adapter *adapter, int v_idx, int t_idx)
349 {
350 struct i40e_q_vector *q_vector = &adapter->q_vectors[v_idx];
351 struct i40e_ring *tx_ring = &adapter->tx_rings[t_idx];
352 struct i40e_hw *hw = &adapter->hw;
354 tx_ring->q_vector = q_vector;
355 tx_ring->next = q_vector->tx.ring;
356 tx_ring->vsi = &adapter->vsi;
357 q_vector->tx.ring = tx_ring;
358 q_vector->tx.count++;
359 q_vector->tx.next_update = jiffies + 1;
360 q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
361 q_vector->num_ringpairs++;
362 wr32(hw, I40E_VFINT_ITRN1(I40E_TX_ITR, q_vector->reg_idx),
363 q_vector->tx.target_itr);
364 q_vector->tx.current_itr = q_vector->tx.target_itr;
365 }
367 /**
368 * i40evf_map_rings_to_vectors - Maps descriptor rings to vectors
369 * @adapter: board private structure to initialize
370 *
371 * This function maps descriptor rings to the queue-specific vectors
372 * we were allotted through the MSI-X enabling code. Ideally, we'd have
373 * one vector per ring/queue, but on a constrained vector budget, we
374 * group the rings as "efficiently" as possible. You would add new
375 * mapping configurations in here.
376 **/
377 static void i40evf_map_rings_to_vectors(struct i40evf_adapter *adapter)
378 {
379 int rings_remaining = adapter->num_active_queues;
380 int ridx = 0, vidx = 0;
381 int q_vectors;
383 q_vectors = adapter->num_msix_vectors - NONQ_VECS;
385 for (; ridx < rings_remaining; ridx++) {
386 i40evf_map_vector_to_rxq(adapter, vidx, ridx);
387 i40evf_map_vector_to_txq(adapter, vidx, ridx);
389 /* In the case where we have more queues than vectors, continue
390 * round-robin on vectors until all queues are mapped.
391 */
392 if (++vidx >= q_vectors)
393 vidx = 0;
394 }
396 adapter->aq_required |= I40EVF_FLAG_AQ_MAP_VECTORS;
397 }
399 /**
400 * i40evf_irq_affinity_notify - Callback for affinity changes
401 * @notify: context as to what irq was changed
402 * @mask: the new affinity mask
403 *
404 * This is a callback function used by the irq_set_affinity_notifier function
405 * so that we may register to receive changes to the irq affinity masks.
406 **/
407 static void i40evf_irq_affinity_notify(struct irq_affinity_notify *notify,
408 const cpumask_t *mask)
409 {
410 struct i40e_q_vector *q_vector =
411 container_of(notify, struct i40e_q_vector, affinity_notify);
413 cpumask_copy(&q_vector->affinity_mask, mask);
414 }
416 /**
417 * i40evf_irq_affinity_release - Callback for affinity notifier release
418 * @ref: internal core kernel usage
419 *
420 * This is a callback function used by the irq_set_affinity_notifier function
421 * to inform the current notification subscriber that they will no longer
422 * receive notifications.
423 **/
424 static void i40evf_irq_affinity_release(struct kref *ref) {}
426 /**
427 * i40evf_request_traffic_irqs - Initialize MSI-X interrupts
428 * @adapter: board private structure
429 * @basename: device basename
430 *
431 * Allocates MSI-X vectors for tx and rx handling, and requests
432 * interrupts from the kernel.
433 **/
434 static int
435 i40evf_request_traffic_irqs(struct i40evf_adapter *adapter, char *basename)
436 {
437 unsigned int vector, q_vectors;
438 unsigned int rx_int_idx = 0, tx_int_idx = 0;
439 int irq_num, err;
440 int cpu;
442 i40evf_irq_disable(adapter);
443 /* Decrement for Other and TCP Timer vectors */
444 q_vectors = adapter->num_msix_vectors - NONQ_VECS;
446 for (vector = 0; vector < q_vectors; vector++) {
447 struct i40e_q_vector *q_vector = &adapter->q_vectors[vector];
448 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
450 if (q_vector->tx.ring && q_vector->rx.ring) {
451 snprintf(q_vector->name, sizeof(q_vector->name),
452 "i40evf-%s-TxRx-%d", basename, rx_int_idx++);
453 tx_int_idx++;
454 } else if (q_vector->rx.ring) {
455 snprintf(q_vector->name, sizeof(q_vector->name),
456 "i40evf-%s-rx-%d", basename, rx_int_idx++);
457 } else if (q_vector->tx.ring) {
458 snprintf(q_vector->name, sizeof(q_vector->name),
459 "i40evf-%s-tx-%d", basename, tx_int_idx++);
460 } else {
461 /* skip this unused q_vector */
462 continue;
463 }
464 err = request_irq(irq_num,
465 i40evf_msix_clean_rings,
466 0,
467 q_vector->name,
468 q_vector);
469 if (err) {
470 dev_info(&adapter->pdev->dev,
471 "Request_irq failed, error: %d\n", err);
472 goto free_queue_irqs;
473 }
474 /* register for affinity change notifications */
475 q_vector->affinity_notify.notify = i40evf_irq_affinity_notify;
476 q_vector->affinity_notify.release =
477 i40evf_irq_affinity_release;
478 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
479 /* Spread the IRQ affinity hints across online CPUs. Note that
480 * get_cpu_mask returns a mask with a permanent lifetime so
481 * it's safe to use as a hint for irq_set_affinity_hint.
482 */
483 cpu = cpumask_local_spread(q_vector->v_idx, -1);
484 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu));
485 }
487 return 0;
489 free_queue_irqs:
490 while (vector) {
491 vector--;
492 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
493 irq_set_affinity_notifier(irq_num, NULL);
494 irq_set_affinity_hint(irq_num, NULL);
495 free_irq(irq_num, &adapter->q_vectors[vector]);
496 }
497 return err;
498 }
500 /**
501 * i40evf_request_misc_irq - Initialize MSI-X interrupts
502 * @adapter: board private structure
503 *
504 * Allocates MSI-X vector 0 and requests interrupts from the kernel. This
505 * vector is only for the admin queue, and stays active even when the netdev
506 * is closed.
507 **/
508 static int i40evf_request_misc_irq(struct i40evf_adapter *adapter)
509 {
510 struct net_device *netdev = adapter->netdev;
511 int err;
513 snprintf(adapter->misc_vector_name,
514 sizeof(adapter->misc_vector_name) - 1, "i40evf-%s:mbx",
515 dev_name(&adapter->pdev->dev));
516 err = request_irq(adapter->msix_entries[0].vector,
517 &i40evf_msix_aq, 0,
518 adapter->misc_vector_name, netdev);
519 if (err) {
520 dev_err(&adapter->pdev->dev,
521 "request_irq for %s failed: %d\n",
522 adapter->misc_vector_name, err);
523 free_irq(adapter->msix_entries[0].vector, netdev);
524 }
525 return err;
526 }
528 /**
529 * i40evf_free_traffic_irqs - Free MSI-X interrupts
530 * @adapter: board private structure
531 *
532 * Frees all MSI-X vectors other than 0.
533 **/
534 static void i40evf_free_traffic_irqs(struct i40evf_adapter *adapter)
535 {
536 int vector, irq_num, q_vectors;
538 if (!adapter->msix_entries)
539 return;
541 q_vectors = adapter->num_msix_vectors - NONQ_VECS;
543 for (vector = 0; vector < q_vectors; vector++) {
544 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
545 irq_set_affinity_notifier(irq_num, NULL);
546 irq_set_affinity_hint(irq_num, NULL);
547 free_irq(irq_num, &adapter->q_vectors[vector]);
548 }
549 }
551 /**
552 * i40evf_free_misc_irq - Free MSI-X miscellaneous vector
553 * @adapter: board private structure
554 *
555 * Frees MSI-X vector 0.
556 **/
557 static void i40evf_free_misc_irq(struct i40evf_adapter *adapter)
558 {
559 struct net_device *netdev = adapter->netdev;
561 if (!adapter->msix_entries)
562 return;
564 free_irq(adapter->msix_entries[0].vector, netdev);
565 }
567 /**
568 * i40evf_configure_tx - Configure Transmit Unit after Reset
569 * @adapter: board private structure
570 *
571 * Configure the Tx unit of the MAC after a reset.
572 **/
573 static void i40evf_configure_tx(struct i40evf_adapter *adapter)
574 {
575 struct i40e_hw *hw = &adapter->hw;
576 int i;
578 for (i = 0; i < adapter->num_active_queues; i++)
579 adapter->tx_rings[i].tail = hw->hw_addr + I40E_QTX_TAIL1(i);
580 }
582 /**
583 * i40evf_configure_rx - Configure Receive Unit after Reset
584 * @adapter: board private structure
585 *
586 * Configure the Rx unit of the MAC after a reset.
587 **/
588 static void i40evf_configure_rx(struct i40evf_adapter *adapter)
589 {
590 unsigned int rx_buf_len = I40E_RXBUFFER_2048;
591 struct i40e_hw *hw = &adapter->hw;
592 int i;
594 /* Legacy Rx will always default to a 2048 buffer size. */
595 #if (PAGE_SIZE < 8192)
596 if (!(adapter->flags & I40EVF_FLAG_LEGACY_RX)) {
597 struct net_device *netdev = adapter->netdev;
599 /* For jumbo frames on systems with 4K pages we have to use
600 * an order 1 page, so we might as well increase the size
601 * of our Rx buffer to make better use of the available space
602 */
603 rx_buf_len = I40E_RXBUFFER_3072;
605 /* We use a 1536 buffer size for configurations with
606 * standard Ethernet mtu. On x86 this gives us enough room
607 * for shared info and 192 bytes of padding.
608 */
609 if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
610 (netdev->mtu <= ETH_DATA_LEN))
611 rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
612 }
613 #endif
615 for (i = 0; i < adapter->num_active_queues; i++) {
616 adapter->rx_rings[i].tail = hw->hw_addr + I40E_QRX_TAIL1(i);
617 adapter->rx_rings[i].rx_buf_len = rx_buf_len;
619 if (adapter->flags & I40EVF_FLAG_LEGACY_RX)
620 clear_ring_build_skb_enabled(&adapter->rx_rings[i]);
621 else
622 set_ring_build_skb_enabled(&adapter->rx_rings[i]);
623 }
624 }
626 /**
627 * i40evf_find_vlan - Search filter list for specific vlan filter
628 * @adapter: board private structure
629 * @vlan: vlan tag
630 *
631 * Returns ptr to the filter object or NULL. Must be called while holding the
632 * mac_vlan_list_lock.
633 **/
634 static struct
635 i40evf_vlan_filter *i40evf_find_vlan(struct i40evf_adapter *adapter, u16 vlan)
636 {
637 struct i40evf_vlan_filter *f;
639 list_for_each_entry(f, &adapter->vlan_filter_list, list) {
640 if (vlan == f->vlan)
641 return f;
642 }
643 return NULL;
644 }
646 /**
647 * i40evf_add_vlan - Add a vlan filter to the list
648 * @adapter: board private structure
649 * @vlan: VLAN tag
650 *
651 * Returns ptr to the filter object or NULL when no memory available.
652 **/
653 static struct
654 i40evf_vlan_filter *i40evf_add_vlan(struct i40evf_adapter *adapter, u16 vlan)
655 {
656 struct i40evf_vlan_filter *f = NULL;
658 spin_lock_bh(&adapter->mac_vlan_list_lock);
660 f = i40evf_find_vlan(adapter, vlan);
661 if (!f) {
662 f = kzalloc(sizeof(*f), GFP_KERNEL);
663 if (!f)
664 goto clearout;
666 f->vlan = vlan;
668 INIT_LIST_HEAD(&f->list);
669 list_add(&f->list, &adapter->vlan_filter_list);
670 f->add = true;
671 adapter->aq_required |= I40EVF_FLAG_AQ_ADD_VLAN_FILTER;
672 }
674 clearout:
675 spin_unlock_bh(&adapter->mac_vlan_list_lock);
676 return f;
677 }
679 /**
680 * i40evf_del_vlan - Remove a vlan filter from the list
681 * @adapter: board private structure
682 * @vlan: VLAN tag
683 **/
684 static void i40evf_del_vlan(struct i40evf_adapter *adapter, u16 vlan)
685 {
686 struct i40evf_vlan_filter *f;
688 spin_lock_bh(&adapter->mac_vlan_list_lock);
690 f = i40evf_find_vlan(adapter, vlan);
691 if (f) {
692 f->remove = true;
693 adapter->aq_required |= I40EVF_FLAG_AQ_DEL_VLAN_FILTER;
694 }
696 spin_unlock_bh(&adapter->mac_vlan_list_lock);
697 }
699 /**
700 * i40evf_vlan_rx_add_vid - Add a VLAN filter to a device
701 * @netdev: network device struct
702 * @proto: unused protocol data
703 * @vid: VLAN tag
704 **/
705 static int i40evf_vlan_rx_add_vid(struct net_device *netdev,
706 __always_unused __be16 proto, u16 vid)
707 {
708 struct i40evf_adapter *adapter = netdev_priv(netdev);
710 if (!VLAN_ALLOWED(adapter))
711 return -EIO;
712 if (i40evf_add_vlan(adapter, vid) == NULL)
713 return -ENOMEM;
714 return 0;
715 }
717 /**
718 * i40evf_vlan_rx_kill_vid - Remove a VLAN filter from a device
719 * @netdev: network device struct
720 * @proto: unused protocol data
721 * @vid: VLAN tag
722 **/
723 static int i40evf_vlan_rx_kill_vid(struct net_device *netdev,
724 __always_unused __be16 proto, u16 vid)
725 {
726 struct i40evf_adapter *adapter = netdev_priv(netdev);
728 if (VLAN_ALLOWED(adapter)) {
729 i40evf_del_vlan(adapter, vid);
730 return 0;
731 }
732 return -EIO;
733 }
735 /**
736 * i40evf_find_filter - Search filter list for specific mac filter
737 * @adapter: board private structure
738 * @macaddr: the MAC address
739 *
740 * Returns ptr to the filter object or NULL. Must be called while holding the
741 * mac_vlan_list_lock.
742 **/
743 static struct
744 i40evf_mac_filter *i40evf_find_filter(struct i40evf_adapter *adapter,
745 const u8 *macaddr)
746 {
747 struct i40evf_mac_filter *f;
749 if (!macaddr)
750 return NULL;
752 list_for_each_entry(f, &adapter->mac_filter_list, list) {
753 if (ether_addr_equal(macaddr, f->macaddr))
754 return f;
755 }
756 return NULL;
757 }
759 /**
760 * i40e_add_filter - Add a mac filter to the filter list
761 * @adapter: board private structure
762 * @macaddr: the MAC address
763 *
764 * Returns ptr to the filter object or NULL when no memory available.
765 **/
766 static struct
767 i40evf_mac_filter *i40evf_add_filter(struct i40evf_adapter *adapter,
768 const u8 *macaddr)
769 {
770 struct i40evf_mac_filter *f;
772 if (!macaddr)
773 return NULL;
775 f = i40evf_find_filter(adapter, macaddr);
776 if (!f) {
777 f = kzalloc(sizeof(*f), GFP_ATOMIC);
778 if (!f)
779 return f;
781 ether_addr_copy(f->macaddr, macaddr);
783 list_add_tail(&f->list, &adapter->mac_filter_list);
784 f->add = true;
785 adapter->aq_required |= I40EVF_FLAG_AQ_ADD_MAC_FILTER;
786 } else {
787 f->remove = false;
788 }
790 return f;
791 }
793 /**
794 * i40evf_set_mac - NDO callback to set port mac address
795 * @netdev: network interface device structure
796 * @p: pointer to an address structure
797 *
798 * Returns 0 on success, negative on failure
799 **/
800 static int i40evf_set_mac(struct net_device *netdev, void *p)
801 {
802 struct i40evf_adapter *adapter = netdev_priv(netdev);
803 struct i40e_hw *hw = &adapter->hw;
804 struct i40evf_mac_filter *f;
805 struct sockaddr *addr = p;
807 if (!is_valid_ether_addr(addr->sa_data))
808 return -EADDRNOTAVAIL;
810 if (ether_addr_equal(netdev->dev_addr, addr->sa_data))
811 return 0;
813 if (adapter->flags & I40EVF_FLAG_ADDR_SET_BY_PF)
814 return -EPERM;
816 spin_lock_bh(&adapter->mac_vlan_list_lock);
818 f = i40evf_find_filter(adapter, hw->mac.addr);
819 if (f) {
820 f->remove = true;
821 adapter->aq_required |= I40EVF_FLAG_AQ_DEL_MAC_FILTER;
822 }
824 f = i40evf_add_filter(adapter, addr->sa_data);
826 spin_unlock_bh(&adapter->mac_vlan_list_lock);
828 if (f) {
829 ether_addr_copy(hw->mac.addr, addr->sa_data);
830 ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr);
831 }
833 return (f == NULL) ? -ENOMEM : 0;
834 }
836 /**
837 * i40evf_addr_sync - Callback for dev_(mc|uc)_sync to add address
838 * @netdev: the netdevice
839 * @addr: address to add
840 *
841 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
842 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
843 */
844 static int i40evf_addr_sync(struct net_device *netdev, const u8 *addr)
845 {
846 struct i40evf_adapter *adapter = netdev_priv(netdev);
848 if (i40evf_add_filter(adapter, addr))
849 return 0;
850 else
851 return -ENOMEM;
852 }
854 /**
855 * i40evf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
856 * @netdev: the netdevice
857 * @addr: address to add
858 *
859 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
860 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
861 */
862 static int i40evf_addr_unsync(struct net_device *netdev, const u8 *addr)
863 {
864 struct i40evf_adapter *adapter = netdev_priv(netdev);
865 struct i40evf_mac_filter *f;
867 /* Under some circumstances, we might receive a request to delete
868 * our own device address from our uc list. Because we store the
869 * device address in the VSI's MAC/VLAN filter list, we need to ignore
870 * such requests and not delete our device address from this list.
871 */
872 if (ether_addr_equal(addr, netdev->dev_addr))
873 return 0;
875 f = i40evf_find_filter(adapter, addr);
876 if (f) {
877 f->remove = true;
878 adapter->aq_required |= I40EVF_FLAG_AQ_DEL_MAC_FILTER;
879 }
880 return 0;
881 }
883 /**
884 * i40evf_set_rx_mode - NDO callback to set the netdev filters
885 * @netdev: network interface device structure
886 **/
887 static void i40evf_set_rx_mode(struct net_device *netdev)
888 {
889 struct i40evf_adapter *adapter = netdev_priv(netdev);
891 spin_lock_bh(&adapter->mac_vlan_list_lock);
892 __dev_uc_sync(netdev, i40evf_addr_sync, i40evf_addr_unsync);
893 __dev_mc_sync(netdev, i40evf_addr_sync, i40evf_addr_unsync);
894 spin_unlock_bh(&adapter->mac_vlan_list_lock);
896 if (netdev->flags & IFF_PROMISC &&
897 !(adapter->flags & I40EVF_FLAG_PROMISC_ON))
898 adapter->aq_required |= I40EVF_FLAG_AQ_REQUEST_PROMISC;
899 else if (!(netdev->flags & IFF_PROMISC) &&
900 adapter->flags & I40EVF_FLAG_PROMISC_ON)
901 adapter->aq_required |= I40EVF_FLAG_AQ_RELEASE_PROMISC;
903 if (netdev->flags & IFF_ALLMULTI &&
904 !(adapter->flags & I40EVF_FLAG_ALLMULTI_ON))
905 adapter->aq_required |= I40EVF_FLAG_AQ_REQUEST_ALLMULTI;
906 else if (!(netdev->flags & IFF_ALLMULTI) &&
907 adapter->flags & I40EVF_FLAG_ALLMULTI_ON)
908 adapter->aq_required |= I40EVF_FLAG_AQ_RELEASE_ALLMULTI;
909 }
911 /**
912 * i40evf_napi_enable_all - enable NAPI on all queue vectors
913 * @adapter: board private structure
914 **/
915 static void i40evf_napi_enable_all(struct i40evf_adapter *adapter)
916 {
917 int q_idx;
918 struct i40e_q_vector *q_vector;
919 int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
921 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
922 struct napi_struct *napi;
924 q_vector = &adapter->q_vectors[q_idx];
925 napi = &q_vector->napi;
926 napi_enable(napi);
927 }
928 }
930 /**
931 * i40evf_napi_disable_all - disable NAPI on all queue vectors
932 * @adapter: board private structure
933 **/
934 static void i40evf_napi_disable_all(struct i40evf_adapter *adapter)
935 {
936 int q_idx;
937 struct i40e_q_vector *q_vector;
938 int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
940 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
941 q_vector = &adapter->q_vectors[q_idx];
942 napi_disable(&q_vector->napi);
943 }
944 }
946 /**
947 * i40evf_configure - set up transmit and receive data structures
948 * @adapter: board private structure
949 **/
950 static void i40evf_configure(struct i40evf_adapter *adapter)
951 {
952 struct net_device *netdev = adapter->netdev;
953 int i;
955 i40evf_set_rx_mode(netdev);
957 i40evf_configure_tx(adapter);
958 i40evf_configure_rx(adapter);
959 adapter->aq_required |= I40EVF_FLAG_AQ_CONFIGURE_QUEUES;
961 for (i = 0; i < adapter->num_active_queues; i++) {
962 struct i40e_ring *ring = &adapter->rx_rings[i];
964 i40evf_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
965 }
966 }
968 /**
969 * i40evf_up_complete - Finish the last steps of bringing up a connection
970 * @adapter: board private structure
971 *
972 * Expects to be called while holding the __I40EVF_IN_CRITICAL_TASK bit lock.
973 **/
974 static void i40evf_up_complete(struct i40evf_adapter *adapter)
975 {
976 adapter->state = __I40EVF_RUNNING;
977 clear_bit(__I40E_VSI_DOWN, adapter->vsi.state);
979 i40evf_napi_enable_all(adapter);
981 adapter->aq_required |= I40EVF_FLAG_AQ_ENABLE_QUEUES;
982 if (CLIENT_ENABLED(adapter))
983 adapter->flags |= I40EVF_FLAG_CLIENT_NEEDS_OPEN;
984 mod_timer_pending(&adapter->watchdog_timer, jiffies + 1);
985 }
987 /**
988 * i40e_down - Shutdown the connection processing
989 * @adapter: board private structure
990 *
991 * Expects to be called while holding the __I40EVF_IN_CRITICAL_TASK bit lock.
992 **/
993 void i40evf_down(struct i40evf_adapter *adapter)
994 {
995 struct net_device *netdev = adapter->netdev;
996 struct i40evf_vlan_filter *vlf;
997 struct i40evf_mac_filter *f;
998 struct i40evf_cloud_filter *cf;
1000 if (adapter->state <= __I40EVF_DOWN_PENDING)
1001 return;
1003 netif_carrier_off(netdev);
1004 netif_tx_disable(netdev);
1005 adapter->link_up = false;
1006 i40evf_napi_disable_all(adapter);
1007 i40evf_irq_disable(adapter);
1009 spin_lock_bh(&adapter->mac_vlan_list_lock);
1011 /* clear the sync flag on all filters */
1012 __dev_uc_unsync(adapter->netdev, NULL);
1013 __dev_mc_unsync(adapter->netdev, NULL);
1015 /* remove all MAC filters */
1016 list_for_each_entry(f, &adapter->mac_filter_list, list) {
1017 f->remove = true;
1018 }
1020 /* remove all VLAN filters */
1021 list_for_each_entry(vlf, &adapter->vlan_filter_list, list) {
1022 vlf->remove = true;
1023 }
1025 spin_unlock_bh(&adapter->mac_vlan_list_lock);
1027 /* remove all cloud filters */
1028 spin_lock_bh(&adapter->cloud_filter_list_lock);
1029 list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
1030 cf->del = true;
1031 }
1032 spin_unlock_bh(&adapter->cloud_filter_list_lock);
1034 if (!(adapter->flags & I40EVF_FLAG_PF_COMMS_FAILED) &&
1035 adapter->state != __I40EVF_RESETTING) {
1036 /* cancel any current operation */
1037 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1038 /* Schedule operations to close down the HW. Don't wait
1039 * here for this to complete. The watchdog is still running
1040 * and it will take care of this.
1041 */
1042 adapter->aq_required = I40EVF_FLAG_AQ_DEL_MAC_FILTER;
1043 adapter->aq_required |= I40EVF_FLAG_AQ_DEL_VLAN_FILTER;
1044 adapter->aq_required |= I40EVF_FLAG_AQ_DEL_CLOUD_FILTER;
1045 adapter->aq_required |= I40EVF_FLAG_AQ_DISABLE_QUEUES;
1046 }
1048 mod_timer_pending(&adapter->watchdog_timer, jiffies + 1);
1049 }
1051 /**
1052 * i40evf_acquire_msix_vectors - Setup the MSIX capability
1053 * @adapter: board private structure
1054 * @vectors: number of vectors to request
1055 *
1056 * Work with the OS to set up the MSIX vectors needed.
1057 *
1058 * Returns 0 on success, negative on failure
1059 **/
1060 static int
1061 i40evf_acquire_msix_vectors(struct i40evf_adapter *adapter, int vectors)
1062 {
1063 int err, vector_threshold;
1065 /* We'll want at least 3 (vector_threshold):
1066 * 0) Other (Admin Queue and link, mostly)
1067 * 1) TxQ[0] Cleanup
1068 * 2) RxQ[0] Cleanup
1069 */
1070 vector_threshold = MIN_MSIX_COUNT;
1072 /* The more we get, the more we will assign to Tx/Rx Cleanup
1073 * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1074 * Right now, we simply care about how many we'll get; we'll
1075 * set them up later while requesting irq's.
1076 */
1077 err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
1078 vector_threshold, vectors);
1079 if (err < 0) {
1080 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n");
1081 kfree(adapter->msix_entries);
1082 adapter->msix_entries = NULL;
1083 return err;
1084 }
1086 /* Adjust for only the vectors we'll use, which is minimum
1087 * of max_msix_q_vectors + NONQ_VECS, or the number of
1088 * vectors we were allocated.
1089 */
1090 adapter->num_msix_vectors = err;
1091 return 0;
1092 }
1094 /**
1095 * i40evf_free_queues - Free memory for all rings
1096 * @adapter: board private structure to initialize
1097 *
1098 * Free all of the memory associated with queue pairs.
1099 **/
1100 static void i40evf_free_queues(struct i40evf_adapter *adapter)
1101 {
1102 if (!adapter->vsi_res)
1103 return;
1104 adapter->num_active_queues = 0;
1105 kfree(adapter->tx_rings);
1106 adapter->tx_rings = NULL;
1107 kfree(adapter->rx_rings);
1108 adapter->rx_rings = NULL;
1109 }
1111 /**
1112 * i40evf_alloc_queues - Allocate memory for all rings
1113 * @adapter: board private structure to initialize
1114 *
1115 * We allocate one ring per queue at run-time since we don't know the
1116 * number of queues at compile-time. The polling_netdev array is
1117 * intended for Multiqueue, but should work fine with a single queue.
1118 **/
1119 static int i40evf_alloc_queues(struct i40evf_adapter *adapter)
1120 {
1121 int i, num_active_queues;
1123 /* If we're in reset reallocating queues we don't actually know yet for
1124 * certain the PF gave us the number of queues we asked for but we'll
1125 * assume it did. Once basic reset is finished we'll confirm once we
1126 * start negotiating config with PF.
1127 */
1128 if (adapter->num_req_queues)
1129 num_active_queues = adapter->num_req_queues;
1130 else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1131 adapter->num_tc)
1132 num_active_queues = adapter->ch_config.total_qps;
1133 else
1134 num_active_queues = min_t(int,
1135 adapter->vsi_res->num_queue_pairs,
1136 (int)(num_online_cpus()));
1139 adapter->tx_rings = kcalloc(num_active_queues,
1140 sizeof(struct i40e_ring), GFP_KERNEL);
1141 if (!adapter->tx_rings)
1142 goto err_out;
1143 adapter->rx_rings = kcalloc(num_active_queues,
1144 sizeof(struct i40e_ring), GFP_KERNEL);
1145 if (!adapter->rx_rings)
1146 goto err_out;
1148 for (i = 0; i < num_active_queues; i++) {
1149 struct i40e_ring *tx_ring;
1150 struct i40e_ring *rx_ring;
1152 tx_ring = &adapter->tx_rings[i];
1154 tx_ring->queue_index = i;
1155 tx_ring->netdev = adapter->netdev;
1156 tx_ring->dev = &adapter->pdev->dev;
1157 tx_ring->count = adapter->tx_desc_count;
1158 tx_ring->itr_setting = I40E_ITR_TX_DEF;
1159 if (adapter->flags & I40EVF_FLAG_WB_ON_ITR_CAPABLE)
1160 tx_ring->flags |= I40E_TXR_FLAGS_WB_ON_ITR;
1162 rx_ring = &adapter->rx_rings[i];
1163 rx_ring->queue_index = i;
1164 rx_ring->netdev = adapter->netdev;
1165 rx_ring->dev = &adapter->pdev->dev;
1166 rx_ring->count = adapter->rx_desc_count;
1167 rx_ring->itr_setting = I40E_ITR_RX_DEF;
1168 }
1170 adapter->num_active_queues = num_active_queues;
1172 return 0;
1174 err_out:
1175 i40evf_free_queues(adapter);
1176 return -ENOMEM;
1177 }
1179 /**
1180 * i40evf_set_interrupt_capability - set MSI-X or FAIL if not supported
1181 * @adapter: board private structure to initialize
1182 *
1183 * Attempt to configure the interrupts using the best available
1184 * capabilities of the hardware and the kernel.
1185 **/
1186 static int i40evf_set_interrupt_capability(struct i40evf_adapter *adapter)
1187 {
1188 int vector, v_budget;
1189 int pairs = 0;
1190 int err = 0;
1192 if (!adapter->vsi_res) {
1193 err = -EIO;
1194 goto out;
1195 }
1196 pairs = adapter->num_active_queues;
1198 /* It's easy to be greedy for MSI-X vectors, but it really doesn't do
1199 * us much good if we have more vectors than CPUs. However, we already
1200 * limit the total number of queues by the number of CPUs so we do not
1201 * need any further limiting here.
1202 */
1203 v_budget = min_t(int, pairs + NONQ_VECS,
1204 (int)adapter->vf_res->max_vectors);
1206 adapter->msix_entries = kcalloc(v_budget,
1207 sizeof(struct msix_entry), GFP_KERNEL);
1208 if (!adapter->msix_entries) {
1209 err = -ENOMEM;
1210 goto out;
1211 }
1213 for (vector = 0; vector < v_budget; vector++)
1214 adapter->msix_entries[vector].entry = vector;
1216 err = i40evf_acquire_msix_vectors(adapter, v_budget);
1218 out:
1219 netif_set_real_num_rx_queues(adapter->netdev, pairs);
1220 netif_set_real_num_tx_queues(adapter->netdev, pairs);
1221 return err;
1222 }
1224 /**
1225 * i40e_config_rss_aq - Configure RSS keys and lut by using AQ commands
1226 * @adapter: board private structure
1227 *
1228 * Return 0 on success, negative on failure
1229 **/
1230 static int i40evf_config_rss_aq(struct i40evf_adapter *adapter)
1231 {
1232 struct i40e_aqc_get_set_rss_key_data *rss_key =
1233 (struct i40e_aqc_get_set_rss_key_data *)adapter->rss_key;
1234 struct i40e_hw *hw = &adapter->hw;
1235 int ret = 0;
1237 if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
1238 /* bail because we already have a command pending */
1239 dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n",
1240 adapter->current_op);
1241 return -EBUSY;
1242 }
1244 ret = i40evf_aq_set_rss_key(hw, adapter->vsi.id, rss_key);
1245 if (ret) {
1246 dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n",
1247 i40evf_stat_str(hw, ret),
1248 i40evf_aq_str(hw, hw->aq.asq_last_status));
1249 return ret;
1251 }
1253 ret = i40evf_aq_set_rss_lut(hw, adapter->vsi.id, false,
1254 adapter->rss_lut, adapter->rss_lut_size);
1255 if (ret) {
1256 dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n",
1257 i40evf_stat_str(hw, ret),
1258 i40evf_aq_str(hw, hw->aq.asq_last_status));
1259 }
1261 return ret;
1263 }
1265 /**
1266 * i40evf_config_rss_reg - Configure RSS keys and lut by writing registers
1267 * @adapter: board private structure
1268 *
1269 * Returns 0 on success, negative on failure
1270 **/
1271 static int i40evf_config_rss_reg(struct i40evf_adapter *adapter)
1272 {
1273 struct i40e_hw *hw = &adapter->hw;
1274 u32 *dw;
1275 u16 i;
1277 dw = (u32 *)adapter->rss_key;
1278 for (i = 0; i <= adapter->rss_key_size / 4; i++)
1279 wr32(hw, I40E_VFQF_HKEY(i), dw[i]);
1281 dw = (u32 *)adapter->rss_lut;
1282 for (i = 0; i <= adapter->rss_lut_size / 4; i++)
1283 wr32(hw, I40E_VFQF_HLUT(i), dw[i]);
1285 i40e_flush(hw);
1287 return 0;
1288 }
1290 /**
1291 * i40evf_config_rss - Configure RSS keys and lut
1292 * @adapter: board private structure
1293 *
1294 * Returns 0 on success, negative on failure
1295 **/
1296 int i40evf_config_rss(struct i40evf_adapter *adapter)
1297 {
1299 if (RSS_PF(adapter)) {
1300 adapter->aq_required |= I40EVF_FLAG_AQ_SET_RSS_LUT |
1301 I40EVF_FLAG_AQ_SET_RSS_KEY;
1302 return 0;
1303 } else if (RSS_AQ(adapter)) {
1304 return i40evf_config_rss_aq(adapter);
1305 } else {
1306 return i40evf_config_rss_reg(adapter);
1307 }
1308 }
1310 /**
1311 * i40evf_fill_rss_lut - Fill the lut with default values
1312 * @adapter: board private structure
1313 **/
1314 static void i40evf_fill_rss_lut(struct i40evf_adapter *adapter)
1315 {
1316 u16 i;
1318 for (i = 0; i < adapter->rss_lut_size; i++)
1319 adapter->rss_lut[i] = i % adapter->num_active_queues;
1320 }
1322 /**
1323 * i40evf_init_rss - Prepare for RSS
1324 * @adapter: board private structure
1325 *
1326 * Return 0 on success, negative on failure
1327 **/
1328 static int i40evf_init_rss(struct i40evf_adapter *adapter)
1329 {
1330 struct i40e_hw *hw = &adapter->hw;
1331 int ret;
1333 if (!RSS_PF(adapter)) {
1334 /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */
1335 if (adapter->vf_res->vf_cap_flags &
1336 VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)
1337 adapter->hena = I40E_DEFAULT_RSS_HENA_EXPANDED;
1338 else
1339 adapter->hena = I40E_DEFAULT_RSS_HENA;
1341 wr32(hw, I40E_VFQF_HENA(0), (u32)adapter->hena);
1342 wr32(hw, I40E_VFQF_HENA(1), (u32)(adapter->hena >> 32));
1343 }
1345 i40evf_fill_rss_lut(adapter);
1347 netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size);
1348 ret = i40evf_config_rss(adapter);
1350 return ret;
1351 }
1353 /**
1354 * i40evf_alloc_q_vectors - Allocate memory for interrupt vectors
1355 * @adapter: board private structure to initialize
1356 *
1357 * We allocate one q_vector per queue interrupt. If allocation fails we
1358 * return -ENOMEM.
1359 **/
1360 static int i40evf_alloc_q_vectors(struct i40evf_adapter *adapter)
1361 {
1362 int q_idx = 0, num_q_vectors;
1363 struct i40e_q_vector *q_vector;
1365 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1366 adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector),
1367 GFP_KERNEL);
1368 if (!adapter->q_vectors)
1369 return -ENOMEM;
1371 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1372 q_vector = &adapter->q_vectors[q_idx];
1373 q_vector->adapter = adapter;
1374 q_vector->vsi = &adapter->vsi;
1375 q_vector->v_idx = q_idx;
1376 q_vector->reg_idx = q_idx;
1377 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
1378 netif_napi_add(adapter->netdev, &q_vector->napi,
1379 i40evf_napi_poll, NAPI_POLL_WEIGHT);
1380 }
1382 return 0;
1383 }
1385 /**
1386 * i40evf_free_q_vectors - Free memory allocated for interrupt vectors
1387 * @adapter: board private structure to initialize
1388 *
1389 * This function frees the memory allocated to the q_vectors. In addition if
1390 * NAPI is enabled it will delete any references to the NAPI struct prior
1391 * to freeing the q_vector.
1392 **/
1393 static void i40evf_free_q_vectors(struct i40evf_adapter *adapter)
1394 {
1395 int q_idx, num_q_vectors;
1396 int napi_vectors;
1398 if (!adapter->q_vectors)
1399 return;
1401 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1402 napi_vectors = adapter->num_active_queues;
1404 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1405 struct i40e_q_vector *q_vector = &adapter->q_vectors[q_idx];
1406 if (q_idx < napi_vectors)
1407 netif_napi_del(&q_vector->napi);
1408 }
1409 kfree(adapter->q_vectors);
1410 adapter->q_vectors = NULL;
1411 }
1413 /**
1414 * i40evf_reset_interrupt_capability - Reset MSIX setup
1415 * @adapter: board private structure
1416 *
1417 **/
1418 void i40evf_reset_interrupt_capability(struct i40evf_adapter *adapter)
1419 {
1420 if (!adapter->msix_entries)
1421 return;
1423 pci_disable_msix(adapter->pdev);
1424 kfree(adapter->msix_entries);
1425 adapter->msix_entries = NULL;
1426 }
1428 /**
1429 * i40evf_init_interrupt_scheme - Determine if MSIX is supported and init
1430 * @adapter: board private structure to initialize
1431 *
1432 **/
1433 int i40evf_init_interrupt_scheme(struct i40evf_adapter *adapter)
1434 {
1435 int err;
1437 err = i40evf_alloc_queues(adapter);
1438 if (err) {
1439 dev_err(&adapter->pdev->dev,
1440 "Unable to allocate memory for queues\n");
1441 goto err_alloc_queues;
1442 }
1444 rtnl_lock();
1445 err = i40evf_set_interrupt_capability(adapter);
1446 rtnl_unlock();
1447 if (err) {
1448 dev_err(&adapter->pdev->dev,
1449 "Unable to setup interrupt capabilities\n");
1450 goto err_set_interrupt;
1451 }
1453 err = i40evf_alloc_q_vectors(adapter);
1454 if (err) {
1455 dev_err(&adapter->pdev->dev,
1456 "Unable to allocate memory for queue vectors\n");
1457 goto err_alloc_q_vectors;
1458 }
1460 /* If we've made it so far while ADq flag being ON, then we haven't
1461 * bailed out anywhere in middle. And ADq isn't just enabled but actual
1462 * resources have been allocated in the reset path.
1463 * Now we can truly claim that ADq is enabled.
1464 */
1465 if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1466 adapter->num_tc)
1467 dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created",
1468 adapter->num_tc);
1470 dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u",
1471 (adapter->num_active_queues > 1) ? "Enabled" : "Disabled",
1472 adapter->num_active_queues);
1474 return 0;
1475 err_alloc_q_vectors:
1476 i40evf_reset_interrupt_capability(adapter);
1477 err_set_interrupt:
1478 i40evf_free_queues(adapter);
1479 err_alloc_queues:
1480 return err;
1481 }
1483 /**
1484 * i40evf_free_rss - Free memory used by RSS structs
1485 * @adapter: board private structure
1486 **/
1487 static void i40evf_free_rss(struct i40evf_adapter *adapter)
1488 {
1489 kfree(adapter->rss_key);
1490 adapter->rss_key = NULL;
1492 kfree(adapter->rss_lut);
1493 adapter->rss_lut = NULL;
1494 }
1496 /**
1497 * i40evf_reinit_interrupt_scheme - Reallocate queues and vectors
1498 * @adapter: board private structure
1499 *
1500 * Returns 0 on success, negative on failure
1501 **/
1502 static int i40evf_reinit_interrupt_scheme(struct i40evf_adapter *adapter)
1503 {
1504 struct net_device *netdev = adapter->netdev;
1505 int err;
1507 if (netif_running(netdev))
1508 i40evf_free_traffic_irqs(adapter);
1509 i40evf_free_misc_irq(adapter);
1510 i40evf_reset_interrupt_capability(adapter);
1511 i40evf_free_q_vectors(adapter);
1512 i40evf_free_queues(adapter);
1514 err = i40evf_init_interrupt_scheme(adapter);
1515 if (err)
1516 goto err;
1518 netif_tx_stop_all_queues(netdev);
1520 err = i40evf_request_misc_irq(adapter);
1521 if (err)
1522 goto err;
1524 set_bit(__I40E_VSI_DOWN, adapter->vsi.state);
1526 i40evf_map_rings_to_vectors(adapter);
1528 if (RSS_AQ(adapter))
1529 adapter->aq_required |= I40EVF_FLAG_AQ_CONFIGURE_RSS;
1530 else
1531 err = i40evf_init_rss(adapter);
1532 err:
1533 return err;
1534 }
1536 /**
1537 * i40evf_watchdog_timer - Periodic call-back timer
1538 * @data: pointer to adapter disguised as unsigned long
1539 **/
1540 static void i40evf_watchdog_timer(struct timer_list *t)
1541 {
1542 struct i40evf_adapter *adapter = from_timer(adapter, t,
1543 watchdog_timer);
1545 schedule_work(&adapter->watchdog_task);
1546 /* timer will be rescheduled in watchdog task */
1547 }
1549 /**
1550 * i40evf_watchdog_task - Periodic call-back task
1551 * @work: pointer to work_struct
1552 **/
1553 static void i40evf_watchdog_task(struct work_struct *work)
1554 {
1555 struct i40evf_adapter *adapter = container_of(work,
1556 struct i40evf_adapter,
1557 watchdog_task);
1558 struct i40e_hw *hw = &adapter->hw;
1559 u32 reg_val;
1561 if (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section))
1562 goto restart_watchdog;
1564 if (adapter->flags & I40EVF_FLAG_PF_COMMS_FAILED) {
1565 reg_val = rd32(hw, I40E_VFGEN_RSTAT) &
1566 I40E_VFGEN_RSTAT_VFR_STATE_MASK;
1567 if ((reg_val == VIRTCHNL_VFR_VFACTIVE) ||
1568 (reg_val == VIRTCHNL_VFR_COMPLETED)) {
1569 /* A chance for redemption! */
1570 dev_err(&adapter->pdev->dev, "Hardware came out of reset. Attempting reinit.\n");
1571 adapter->state = __I40EVF_STARTUP;
1572 adapter->flags &= ~I40EVF_FLAG_PF_COMMS_FAILED;
1573 schedule_delayed_work(&adapter->init_task, 10);
1574 clear_bit(__I40EVF_IN_CRITICAL_TASK,
1575 &adapter->crit_section);
1576 /* Don't reschedule the watchdog, since we've restarted
1577 * the init task. When init_task contacts the PF and
1578 * gets everything set up again, it'll restart the
1579 * watchdog for us. Down, boy. Sit. Stay. Woof.
1580 */
1581 return;
1582 }
1583 adapter->aq_required = 0;
1584 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1585 goto watchdog_done;
1586 }
1588 if ((adapter->state < __I40EVF_DOWN) ||
1589 (adapter->flags & I40EVF_FLAG_RESET_PENDING))
1590 goto watchdog_done;
1592 /* check for reset */
1593 reg_val = rd32(hw, I40E_VF_ARQLEN1) & I40E_VF_ARQLEN1_ARQENABLE_MASK;
1594 if (!(adapter->flags & I40EVF_FLAG_RESET_PENDING) && !reg_val) {
1595 adapter->state = __I40EVF_RESETTING;
1596 adapter->flags |= I40EVF_FLAG_RESET_PENDING;
1597 dev_err(&adapter->pdev->dev, "Hardware reset detected\n");
1598 schedule_work(&adapter->reset_task);
1599 adapter->aq_required = 0;
1600 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1601 goto watchdog_done;
1602 }
1604 /* Process admin queue tasks. After init, everything gets done
1605 * here so we don't race on the admin queue.
1606 */
1607 if (adapter->current_op) {
1608 if (!i40evf_asq_done(hw)) {
1609 dev_dbg(&adapter->pdev->dev, "Admin queue timeout\n");
1610 i40evf_send_api_ver(adapter);
1611 }
1612 goto watchdog_done;
1613 }
1614 if (adapter->aq_required & I40EVF_FLAG_AQ_GET_CONFIG) {
1615 i40evf_send_vf_config_msg(adapter);
1616 goto watchdog_done;
1617 }
1619 if (adapter->aq_required & I40EVF_FLAG_AQ_DISABLE_QUEUES) {
1620 i40evf_disable_queues(adapter);
1621 goto watchdog_done;
1622 }
1624 if (adapter->aq_required & I40EVF_FLAG_AQ_MAP_VECTORS) {
1625 i40evf_map_queues(adapter);
1626 goto watchdog_done;
1627 }
1629 if (adapter->aq_required & I40EVF_FLAG_AQ_ADD_MAC_FILTER) {
1630 i40evf_add_ether_addrs(adapter);
1631 goto watchdog_done;
1632 }
1634 if (adapter->aq_required & I40EVF_FLAG_AQ_ADD_VLAN_FILTER) {
1635 i40evf_add_vlans(adapter);
1636 goto watchdog_done;
1637 }
1639 if (adapter->aq_required & I40EVF_FLAG_AQ_DEL_MAC_FILTER) {
1640 i40evf_del_ether_addrs(adapter);
1641 goto watchdog_done;
1642 }
1644 if (adapter->aq_required & I40EVF_FLAG_AQ_DEL_VLAN_FILTER) {
1645 i40evf_del_vlans(adapter);
1646 goto watchdog_done;
1647 }
1649 if (adapter->aq_required & I40EVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) {
1650 i40evf_enable_vlan_stripping(adapter);
1651 goto watchdog_done;
1652 }
1654 if (adapter->aq_required & I40EVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) {
1655 i40evf_disable_vlan_stripping(adapter);
1656 goto watchdog_done;
1657 }
1659 if (adapter->aq_required & I40EVF_FLAG_AQ_CONFIGURE_QUEUES) {
1660 i40evf_configure_queues(adapter);
1661 goto watchdog_done;
1662 }
1664 if (adapter->aq_required & I40EVF_FLAG_AQ_ENABLE_QUEUES) {
1665 i40evf_enable_queues(adapter);
1666 goto watchdog_done;
1667 }
1669 if (adapter->aq_required & I40EVF_FLAG_AQ_CONFIGURE_RSS) {
1670 /* This message goes straight to the firmware, not the
1671 * PF, so we don't have to set current_op as we will
1672 * not get a response through the ARQ.
1673 */
1674 i40evf_init_rss(adapter);
1675 adapter->aq_required &= ~I40EVF_FLAG_AQ_CONFIGURE_RSS;
1676 goto watchdog_done;
1677 }
1678 if (adapter->aq_required & I40EVF_FLAG_AQ_GET_HENA) {
1679 i40evf_get_hena(adapter);
1680 goto watchdog_done;
1681 }
1682 if (adapter->aq_required & I40EVF_FLAG_AQ_SET_HENA) {
1683 i40evf_set_hena(adapter);
1684 goto watchdog_done;
1685 }
1686 if (adapter->aq_required & I40EVF_FLAG_AQ_SET_RSS_KEY) {
1687 i40evf_set_rss_key(adapter);
1688 goto watchdog_done;
1689 }
1690 if (adapter->aq_required & I40EVF_FLAG_AQ_SET_RSS_LUT) {
1691 i40evf_set_rss_lut(adapter);
1692 goto watchdog_done;
1693 }
1695 if (adapter->aq_required & I40EVF_FLAG_AQ_REQUEST_PROMISC) {
1696 i40evf_set_promiscuous(adapter, FLAG_VF_UNICAST_PROMISC |
1697 FLAG_VF_MULTICAST_PROMISC);
1698 goto watchdog_done;
1699 }
1701 if (adapter->aq_required & I40EVF_FLAG_AQ_REQUEST_ALLMULTI) {
1702 i40evf_set_promiscuous(adapter, FLAG_VF_MULTICAST_PROMISC);
1703 goto watchdog_done;
1704 }
1706 if ((adapter->aq_required & I40EVF_FLAG_AQ_RELEASE_PROMISC) &&
1707 (adapter->aq_required & I40EVF_FLAG_AQ_RELEASE_ALLMULTI)) {
1708 i40evf_set_promiscuous(adapter, 0);
1709 goto watchdog_done;
1710 }
1712 if (adapter->aq_required & I40EVF_FLAG_AQ_ENABLE_CHANNELS) {
1713 i40evf_enable_channels(adapter);
1714 goto watchdog_done;
1715 }
1717 if (adapter->aq_required & I40EVF_FLAG_AQ_DISABLE_CHANNELS) {
1718 i40evf_disable_channels(adapter);
1719 goto watchdog_done;
1720 }
1722 if (adapter->aq_required & I40EVF_FLAG_AQ_ADD_CLOUD_FILTER) {
1723 i40evf_add_cloud_filter(adapter);
1724 goto watchdog_done;
1725 }
1727 if (adapter->aq_required & I40EVF_FLAG_AQ_DEL_CLOUD_FILTER) {
1728 i40evf_del_cloud_filter(adapter);
1729 goto watchdog_done;
1730 }
1732 schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5));
1734 if (adapter->state == __I40EVF_RUNNING)
1735 i40evf_request_stats(adapter);
1736 watchdog_done:
1737 if (adapter->state == __I40EVF_RUNNING)
1738 i40evf_detect_recover_hung(&adapter->vsi);
1739 clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
1740 restart_watchdog:
1741 if (adapter->state == __I40EVF_REMOVE)
1742 return;
1743 if (adapter->aq_required)
1744 mod_timer(&adapter->watchdog_timer,
1745 jiffies + msecs_to_jiffies(20));
1746 else
1747 mod_timer(&adapter->watchdog_timer, jiffies + (HZ * 2));
1748 schedule_work(&adapter->adminq_task);
1749 }
1751 static void i40evf_disable_vf(struct i40evf_adapter *adapter)
1752 {
1753 struct i40evf_mac_filter *f, *ftmp;
1754 struct i40evf_vlan_filter *fv, *fvtmp;
1755 struct i40evf_cloud_filter *cf, *cftmp;
1757 adapter->flags |= I40EVF_FLAG_PF_COMMS_FAILED;
1759 /* We don't use netif_running() because it may be true prior to
1760 * ndo_open() returning, so we can't assume it means all our open
1761 * tasks have finished, since we're not holding the rtnl_lock here.
1762 */
1763 if (adapter->state == __I40EVF_RUNNING) {
1764 set_bit(__I40E_VSI_DOWN, adapter->vsi.state);
1765 netif_carrier_off(adapter->netdev);
1766 netif_tx_disable(adapter->netdev);
1767 adapter->link_up = false;
1768 i40evf_napi_disable_all(adapter);
1769 i40evf_irq_disable(adapter);
1770 i40evf_free_traffic_irqs(adapter);
1771 i40evf_free_all_tx_resources(adapter);
1772 i40evf_free_all_rx_resources(adapter);
1773 }
1775 spin_lock_bh(&adapter->mac_vlan_list_lock);
1777 /* Delete all of the filters */
1778 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
1779 list_del(&f->list);
1780 kfree(f);
1781 }
1783 list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) {
1784 list_del(&fv->list);
1785 kfree(fv);
1786 }
1788 spin_unlock_bh(&adapter->mac_vlan_list_lock);
1790 spin_lock_bh(&adapter->cloud_filter_list_lock);
1791 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
1792 list_del(&cf->list);
1793 kfree(cf);
1794 adapter->num_cloud_filters--;
1795 }
1796 spin_unlock_bh(&adapter->cloud_filter_list_lock);
1798 i40evf_free_misc_irq(adapter);
1799 i40evf_reset_interrupt_capability(adapter);
1800 i40evf_free_queues(adapter);
1801 i40evf_free_q_vectors(adapter);
1802 kfree(adapter->vf_res);
1803 i40evf_shutdown_adminq(&adapter->hw);
1804 adapter->netdev->flags &= ~IFF_UP;
1805 clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
1806 adapter->flags &= ~I40EVF_FLAG_RESET_PENDING;
1807 adapter->state = __I40EVF_DOWN;
1808 wake_up(&adapter->down_waitqueue);
1809 dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");
1810 }
1812 #define I40EVF_RESET_WAIT_MS 10
1813 #define I40EVF_RESET_WAIT_COUNT 500
1814 /**
1815 * i40evf_reset_task - Call-back task to handle hardware reset
1816 * @work: pointer to work_struct
1817 *
1818 * During reset we need to shut down and reinitialize the admin queue
1819 * before we can use it to communicate with the PF again. We also clear
1820 * and reinit the rings because that context is lost as well.
1821 **/
1822 static void i40evf_reset_task(struct work_struct *work)
1823 {
1824 struct i40evf_adapter *adapter = container_of(work,
1825 struct i40evf_adapter,
1826 reset_task);
1827 struct virtchnl_vf_resource *vfres = adapter->vf_res;
1828 struct net_device *netdev = adapter->netdev;
1829 struct i40e_hw *hw = &adapter->hw;
1830 struct i40evf_vlan_filter *vlf;
1831 struct i40evf_cloud_filter *cf;
1832 struct i40evf_mac_filter *f;
1833 u32 reg_val;
1834 int i = 0, err;
1835 bool running;
1837 /* When device is being removed it doesn't make sense to run the reset
1838 * task, just return in such a case.
1839 */
1840 if (test_bit(__I40EVF_IN_REMOVE_TASK, &adapter->crit_section))
1841 return;
1843 while (test_and_set_bit(__I40EVF_IN_CLIENT_TASK,
1844 &adapter->crit_section))
1845 usleep_range(500, 1000);
1846 if (CLIENT_ENABLED(adapter)) {
1847 adapter->flags &= ~(I40EVF_FLAG_CLIENT_NEEDS_OPEN |
1848 I40EVF_FLAG_CLIENT_NEEDS_CLOSE |
1849 I40EVF_FLAG_CLIENT_NEEDS_L2_PARAMS |
1850 I40EVF_FLAG_SERVICE_CLIENT_REQUESTED);
1851 cancel_delayed_work_sync(&adapter->client_task);
1852 i40evf_notify_client_close(&adapter->vsi, true);
1853 }
1854 i40evf_misc_irq_disable(adapter);
1855 if (adapter->flags & I40EVF_FLAG_RESET_NEEDED) {
1856 adapter->flags &= ~I40EVF_FLAG_RESET_NEEDED;
1857 /* Restart the AQ here. If we have been reset but didn't
1858 * detect it, or if the PF had to reinit, our AQ will be hosed.
1859 */
1860 i40evf_shutdown_adminq(hw);
1861 i40evf_init_adminq(hw);
1862 i40evf_request_reset(adapter);
1863 }
1864 adapter->flags |= I40EVF_FLAG_RESET_PENDING;
1866 /* poll until we see the reset actually happen */
1867 for (i = 0; i < I40EVF_RESET_WAIT_COUNT; i++) {
1868 reg_val = rd32(hw, I40E_VF_ARQLEN1) &
1869 I40E_VF_ARQLEN1_ARQENABLE_MASK;
1870 if (!reg_val)
1871 break;
1872 usleep_range(5000, 10000);
1873 }
1874 if (i == I40EVF_RESET_WAIT_COUNT) {
1875 dev_info(&adapter->pdev->dev, "Never saw reset\n");
1876 goto continue_reset; /* act like the reset happened */
1877 }
1879 /* wait until the reset is complete and the PF is responding to us */
1880 for (i = 0; i < I40EVF_RESET_WAIT_COUNT; i++) {
1881 /* sleep first to make sure a minimum wait time is met */
1882 msleep(I40EVF_RESET_WAIT_MS);
1884 reg_val = rd32(hw, I40E_VFGEN_RSTAT) &
1885 I40E_VFGEN_RSTAT_VFR_STATE_MASK;
1886 if (reg_val == VIRTCHNL_VFR_VFACTIVE)
1887 break;
1888 }
1890 pci_set_master(adapter->pdev);
1892 if (i == I40EVF_RESET_WAIT_COUNT) {
1893 dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n",
1894 reg_val);
1895 i40evf_disable_vf(adapter);
1896 clear_bit(__I40EVF_IN_CLIENT_TASK, &adapter->crit_section);
1897 return; /* Do not attempt to reinit. It's dead, Jim. */
1898 }
1900 continue_reset:
1901 /* We don't use netif_running() because it may be true prior to
1902 * ndo_open() returning, so we can't assume it means all our open
1903 * tasks have finished, since we're not holding the rtnl_lock here.
1904 */
1905 running = ((adapter->state == __I40EVF_RUNNING) ||
1906 (adapter->state == __I40EVF_RESETTING));
1908 if (running) {
1909 netif_carrier_off(netdev);
1910 netif_tx_stop_all_queues(netdev);
1911 adapter->link_up = false;
1912 i40evf_napi_disable_all(adapter);
1913 }
1914 i40evf_irq_disable(adapter);
1916 adapter->state = __I40EVF_RESETTING;
1917 adapter->flags &= ~I40EVF_FLAG_RESET_PENDING;
1919 /* free the Tx/Rx rings and descriptors, might be better to just
1920 * re-use them sometime in the future
1921 */
1922 i40evf_free_all_rx_resources(adapter);
1923 i40evf_free_all_tx_resources(adapter);
1925 adapter->flags |= I40EVF_FLAG_QUEUES_DISABLED;
1926 /* kill and reinit the admin queue */
1927 i40evf_shutdown_adminq(hw);
1928 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1929 err = i40evf_init_adminq(hw);
1930 if (err)
1931 dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n",
1932 err);
1933 adapter->aq_required = 0;
1935 if (adapter->flags & I40EVF_FLAG_REINIT_ITR_NEEDED) {
1936 err = i40evf_reinit_interrupt_scheme(adapter);
1937 if (err)
1938 goto reset_err;
1939 }
1941 adapter->aq_required |= I40EVF_FLAG_AQ_GET_CONFIG;
1942 adapter->aq_required |= I40EVF_FLAG_AQ_MAP_VECTORS;
1944 spin_lock_bh(&adapter->mac_vlan_list_lock);
1946 /* re-add all MAC filters */
1947 list_for_each_entry(f, &adapter->mac_filter_list, list) {
1948 f->add = true;
1949 }
1950 /* re-add all VLAN filters */
1951 list_for_each_entry(vlf, &adapter->vlan_filter_list, list) {
1952 vlf->add = true;
1953 }
1955 spin_unlock_bh(&adapter->mac_vlan_list_lock);
1957 /* check if TCs are running and re-add all cloud filters */
1958 spin_lock_bh(&adapter->cloud_filter_list_lock);
1959 if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1960 adapter->num_tc) {
1961 list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
1962 cf->add = true;
1963 }
1964 }
1965 spin_unlock_bh(&adapter->cloud_filter_list_lock);
1967 adapter->aq_required |= I40EVF_FLAG_AQ_ADD_MAC_FILTER;
1968 adapter->aq_required |= I40EVF_FLAG_AQ_ADD_VLAN_FILTER;
1969 adapter->aq_required |= I40EVF_FLAG_AQ_ADD_CLOUD_FILTER;
1970 i40evf_misc_irq_enable(adapter);
1972 mod_timer(&adapter->watchdog_timer, jiffies + 2);
1974 /* We were running when the reset started, so we need to restore some
1975 * state here.
1976 */
1977 if (running) {
1978 /* allocate transmit descriptors */
1979 err = i40evf_setup_all_tx_resources(adapter);
1980 if (err)
1981 goto reset_err;
1983 /* allocate receive descriptors */
1984 err = i40evf_setup_all_rx_resources(adapter);
1985 if (err)
1986 goto reset_err;
1988 if (adapter->flags & I40EVF_FLAG_REINIT_ITR_NEEDED) {
1989 err = i40evf_request_traffic_irqs(adapter,
1990 netdev->name);
1991 if (err)
1992 goto reset_err;
1994 adapter->flags &= ~I40EVF_FLAG_REINIT_ITR_NEEDED;
1995 }
1997 i40evf_configure(adapter);
1999 i40evf_up_complete(adapter);
2001 i40evf_irq_enable(adapter, true);
2002 } else {
2003 adapter->state = __I40EVF_DOWN;
2004 wake_up(&adapter->down_waitqueue);
2005 }
2006 clear_bit(__I40EVF_IN_CLIENT_TASK, &adapter->crit_section);
2007 clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
2009 return;
2010 reset_err:
2011 clear_bit(__I40EVF_IN_CLIENT_TASK, &adapter->crit_section);
2012 clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
2013 dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
2014 i40evf_close(netdev);
2015 }
2017 /**
2018 * i40evf_adminq_task - worker thread to clean the admin queue
2019 * @work: pointer to work_struct containing our data
2020 **/
2021 static void i40evf_adminq_task(struct work_struct *work)
2022 {
2023 struct i40evf_adapter *adapter =
2024 container_of(work, struct i40evf_adapter, adminq_task);
2025 struct i40e_hw *hw = &adapter->hw;
2026 struct i40e_arq_event_info event;
2027 enum virtchnl_ops v_op;
2028 i40e_status ret, v_ret;
2029 u32 val, oldval;
2030 u16 pending;
2032 if (adapter->flags & I40EVF_FLAG_PF_COMMS_FAILED)
2033 goto out;
2035 event.buf_len = I40EVF_MAX_AQ_BUF_SIZE;
2036 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
2037 if (!event.msg_buf)
2038 goto out;
2040 do {
2041 ret = i40evf_clean_arq_element(hw, &event, &pending);
2042 v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high);
2043 v_ret = (i40e_status)le32_to_cpu(event.desc.cookie_low);
2045 if (ret || !v_op)
2046 break; /* No event to process or error cleaning ARQ */
2048 i40evf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf,
2049 event.msg_len);
2050 if (pending != 0)
2051 memset(event.msg_buf, 0, I40EVF_MAX_AQ_BUF_SIZE);
2052 } while (pending);
2054 if ((adapter->flags &
2055 (I40EVF_FLAG_RESET_PENDING | I40EVF_FLAG_RESET_NEEDED)) ||
2056 adapter->state == __I40EVF_RESETTING)
2057 goto freedom;
2059 /* check for error indications */
2060 val = rd32(hw, hw->aq.arq.len);
2061 if (val == 0xdeadbeef) /* indicates device in reset */
2062 goto freedom;
2063 oldval = val;
2064 if (val & I40E_VF_ARQLEN1_ARQVFE_MASK) {
2065 dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n");
2066 val &= ~I40E_VF_ARQLEN1_ARQVFE_MASK;
2067 }
2068 if (val & I40E_VF_ARQLEN1_ARQOVFL_MASK) {
2069 dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n");
2070 val &= ~I40E_VF_ARQLEN1_ARQOVFL_MASK;
2071 }
2072 if (val & I40E_VF_ARQLEN1_ARQCRIT_MASK) {
2073 dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n");
2074 val &= ~I40E_VF_ARQLEN1_ARQCRIT_MASK;
2075 }
2076 if (oldval != val)
2077 wr32(hw, hw->aq.arq.len, val);
2079 val = rd32(hw, hw->aq.asq.len);
2080 oldval = val;
2081 if (val & I40E_VF_ATQLEN1_ATQVFE_MASK) {
2082 dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n");
2083 val &= ~I40E_VF_ATQLEN1_ATQVFE_MASK;
2084 }
2085 if (val & I40E_VF_ATQLEN1_ATQOVFL_MASK) {
2086 dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n");
2087 val &= ~I40E_VF_ATQLEN1_ATQOVFL_MASK;
2088 }
2089 if (val & I40E_VF_ATQLEN1_ATQCRIT_MASK) {
2090 dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n");
2091 val &= ~I40E_VF_ATQLEN1_ATQCRIT_MASK;
2092 }
2093 if (oldval != val)
2094 wr32(hw, hw->aq.asq.len, val);
2096 freedom:
2097 kfree(event.msg_buf);
2098 out:
2099 /* re-enable Admin queue interrupt cause */
2100 i40evf_misc_irq_enable(adapter);
2101 }
2103 /**
2104 * i40evf_client_task - worker thread to perform client work
2105 * @work: pointer to work_struct containing our data
2106 *
2107 * This task handles client interactions. Because client calls can be
2108 * reentrant, we can't handle them in the watchdog.
2109 **/
2110 static void i40evf_client_task(struct work_struct *work)
2111 {
2112 struct i40evf_adapter *adapter =
2113 container_of(work, struct i40evf_adapter, client_task.work);
2115 /* If we can't get the client bit, just give up. We'll be rescheduled
2116 * later.
2117 */
2119 if (test_and_set_bit(__I40EVF_IN_CLIENT_TASK, &adapter->crit_section))
2120 return;
2122 if (adapter->flags & I40EVF_FLAG_SERVICE_CLIENT_REQUESTED) {
2123 i40evf_client_subtask(adapter);
2124 adapter->flags &= ~I40EVF_FLAG_SERVICE_CLIENT_REQUESTED;
2125 goto out;
2126 }
2127 if (adapter->flags & I40EVF_FLAG_CLIENT_NEEDS_L2_PARAMS) {
2128 i40evf_notify_client_l2_params(&adapter->vsi);
2129 adapter->flags &= ~I40EVF_FLAG_CLIENT_NEEDS_L2_PARAMS;
2130 goto out;
2131 }
2132 if (adapter->flags & I40EVF_FLAG_CLIENT_NEEDS_CLOSE) {
2133 i40evf_notify_client_close(&adapter->vsi, false);
2134 adapter->flags &= ~I40EVF_FLAG_CLIENT_NEEDS_CLOSE;
2135 goto out;
2136 }
2137 if (adapter->flags & I40EVF_FLAG_CLIENT_NEEDS_OPEN) {
2138 i40evf_notify_client_open(&adapter->vsi);
2139 adapter->flags &= ~I40EVF_FLAG_CLIENT_NEEDS_OPEN;
2140 }
2141 out:
2142 clear_bit(__I40EVF_IN_CLIENT_TASK, &adapter->crit_section);
2143 }
2145 /**
2146 * i40evf_free_all_tx_resources - Free Tx Resources for All Queues
2147 * @adapter: board private structure
2148 *
2149 * Free all transmit software resources
2150 **/
2151 void i40evf_free_all_tx_resources(struct i40evf_adapter *adapter)
2152 {
2153 int i;
2155 if (!adapter->tx_rings)
2156 return;
2158 for (i = 0; i < adapter->num_active_queues; i++)
2159 if (adapter->tx_rings[i].desc)
2160 i40evf_free_tx_resources(&adapter->tx_rings[i]);
2161 }
2163 /**
2164 * i40evf_setup_all_tx_resources - allocate all queues Tx resources
2165 * @adapter: board private structure
2166 *
2167 * If this function returns with an error, then it's possible one or
2168 * more of the rings is populated (while the rest are not). It is the
2169 * callers duty to clean those orphaned rings.
2170 *
2171 * Return 0 on success, negative on failure
2172 **/
2173 static int i40evf_setup_all_tx_resources(struct i40evf_adapter *adapter)
2174 {
2175 int i, err = 0;
2177 for (i = 0; i < adapter->num_active_queues; i++) {
2178 adapter->tx_rings[i].count = adapter->tx_desc_count;
2179 err = i40evf_setup_tx_descriptors(&adapter->tx_rings[i]);
2180 if (!err)
2181 continue;
2182 dev_err(&adapter->pdev->dev,
2183 "Allocation for Tx Queue %u failed\n", i);
2184 break;
2185 }
2187 return err;
2188 }
2190 /**
2191 * i40evf_setup_all_rx_resources - allocate all queues Rx resources
2192 * @adapter: board private structure
2193 *
2194 * If this function returns with an error, then it's possible one or
2195 * more of the rings is populated (while the rest are not). It is the
2196 * callers duty to clean those orphaned rings.
2197 *
2198 * Return 0 on success, negative on failure
2199 **/
2200 static int i40evf_setup_all_rx_resources(struct i40evf_adapter *adapter)
2201 {
2202 int i, err = 0;
2204 for (i = 0; i < adapter->num_active_queues; i++) {
2205 adapter->rx_rings[i].count = adapter->rx_desc_count;
2206 err = i40evf_setup_rx_descriptors(&adapter->rx_rings[i]);
2207 if (!err)
2208 continue;
2209 dev_err(&adapter->pdev->dev,
2210 "Allocation for Rx Queue %u failed\n", i);
2211 break;
2212 }
2213 return err;
2214 }
2216 /**
2217 * i40evf_free_all_rx_resources - Free Rx Resources for All Queues
2218 * @adapter: board private structure
2219 *
2220 * Free all receive software resources
2221 **/
2222 void i40evf_free_all_rx_resources(struct i40evf_adapter *adapter)
2223 {
2224 int i;
2226 if (!adapter->rx_rings)
2227 return;
2229 for (i = 0; i < adapter->num_active_queues; i++)
2230 if (adapter->rx_rings[i].desc)
2231 i40evf_free_rx_resources(&adapter->rx_rings[i]);
2232 }
2234 /**
2235 * i40evf_validate_tx_bandwidth - validate the max Tx bandwidth
2236 * @adapter: board private structure
2237 * @max_tx_rate: max Tx bw for a tc
2238 **/
2239 static int i40evf_validate_tx_bandwidth(struct i40evf_adapter *adapter,
2240 u64 max_tx_rate)
2241 {
2242 int speed = 0, ret = 0;
2244 switch (adapter->link_speed) {
2245 case I40E_LINK_SPEED_40GB:
2246 speed = 40000;
2247 break;
2248 case I40E_LINK_SPEED_25GB:
2249 speed = 25000;
2250 break;
2251 case I40E_LINK_SPEED_20GB:
2252 speed = 20000;
2253 break;
2254 case I40E_LINK_SPEED_10GB:
2255 speed = 10000;
2256 break;
2257 case I40E_LINK_SPEED_1GB:
2258 speed = 1000;
2259 break;
2260 case I40E_LINK_SPEED_100MB:
2261 speed = 100;
2262 break;
2263 default:
2264 break;
2265 }
2267 if (max_tx_rate > speed) {
2268 dev_err(&adapter->pdev->dev,
2269 "Invalid tx rate specified\n");
2270 ret = -EINVAL;
2271 }
2273 return ret;
2274 }
2276 /**
2277 * i40evf_validate_channel_config - validate queue mapping info
2278 * @adapter: board private structure
2279 * @mqprio_qopt: queue parameters
2280 *
2281 * This function validates if the config provided by the user to
2282 * configure queue channels is valid or not. Returns 0 on a valid
2283 * config.
2284 **/
2285 static int i40evf_validate_ch_config(struct i40evf_adapter *adapter,
2286 struct tc_mqprio_qopt_offload *mqprio_qopt)
2287 {
2288 u64 total_max_rate = 0;
2289 int i, num_qps = 0;
2290 u64 tx_rate = 0;
2291 int ret = 0;
2293 if (mqprio_qopt->qopt.num_tc > I40EVF_MAX_TRAFFIC_CLASS ||
2294 mqprio_qopt->qopt.num_tc < 1)
2295 return -EINVAL;
2297 for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) {
2298 if (!mqprio_qopt->qopt.count[i] ||
2299 mqprio_qopt->qopt.offset[i] != num_qps)
2300 return -EINVAL;
2301 if (mqprio_qopt->min_rate[i]) {
2302 dev_err(&adapter->pdev->dev,
2303 "Invalid min tx rate (greater than 0) specified\n");
2304 return -EINVAL;
2305 }
2306 /*convert to Mbps */
2307 tx_rate = div_u64(mqprio_qopt->max_rate[i],
2308 I40EVF_MBPS_DIVISOR);
2309 total_max_rate += tx_rate;
2310 num_qps += mqprio_qopt->qopt.count[i];
2311 }
2312 if (num_qps > I40EVF_MAX_REQ_QUEUES)
2313 return -EINVAL;
2315 ret = i40evf_validate_tx_bandwidth(adapter, total_max_rate);
2316 return ret;
2317 }
2319 /**
2320 * i40evf_del_all_cloud_filters - delete all cloud filters
2321 * on the traffic classes
2322 **/
2323 static void i40evf_del_all_cloud_filters(struct i40evf_adapter *adapter)
2324 {
2325 struct i40evf_cloud_filter *cf, *cftmp;
2327 spin_lock_bh(&adapter->cloud_filter_list_lock);
2328 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
2329 list) {
2330 list_del(&cf->list);
2331 kfree(cf);
2332 adapter->num_cloud_filters--;
2333 }
2334 spin_unlock_bh(&adapter->cloud_filter_list_lock);
2335 }
2337 /**
2338 * __i40evf_setup_tc - configure multiple traffic classes
2339 * @netdev: network interface device structure
2340 * @type_date: tc offload data
2341 *
2342 * This function processes the config information provided by the
2343 * user to configure traffic classes/queue channels and packages the
2344 * information to request the PF to setup traffic classes.
2345 *
2346 * Returns 0 on success.
2347 **/
2348 static int __i40evf_setup_tc(struct net_device *netdev, void *type_data)
2349 {
2350 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
2351 struct i40evf_adapter *adapter = netdev_priv(netdev);
2352 struct virtchnl_vf_resource *vfres = adapter->vf_res;
2353 u8 num_tc = 0, total_qps = 0;
2354 int ret = 0, netdev_tc = 0;
2355 u64 max_tx_rate;
2356 u16 mode;
2357 int i;
2359 num_tc = mqprio_qopt->qopt.num_tc;
2360 mode = mqprio_qopt->mode;
2362 /* delete queue_channel */
2363 if (!mqprio_qopt->qopt.hw) {
2364 if (adapter->ch_config.state == __I40EVF_TC_RUNNING) {
2365 /* reset the tc configuration */
2366 netdev_reset_tc(netdev);
2367 adapter->num_tc = 0;
2368 netif_tx_stop_all_queues(netdev);
2369 netif_tx_disable(netdev);
2370 i40evf_del_all_cloud_filters(adapter);
2371 adapter->aq_required = I40EVF_FLAG_AQ_DISABLE_CHANNELS;
2372 goto exit;
2373 } else {
2374 return -EINVAL;
2375 }
2376 }
2378 /* add queue channel */
2379 if (mode == TC_MQPRIO_MODE_CHANNEL) {
2380 if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) {
2381 dev_err(&adapter->pdev->dev, "ADq not supported\n");
2382 return -EOPNOTSUPP;
2383 }
2384 if (adapter->ch_config.state != __I40EVF_TC_INVALID) {
2385 dev_err(&adapter->pdev->dev, "TC configuration already exists\n");
2386 return -EINVAL;
2387 }
2389 ret = i40evf_validate_ch_config(adapter, mqprio_qopt);
2390 if (ret)
2391 return ret;
2392 /* Return if same TC config is requested */
2393 if (adapter->num_tc == num_tc)
2394 return 0;
2395 adapter->num_tc = num_tc;
2397 for (i = 0; i < I40EVF_MAX_TRAFFIC_CLASS; i++) {
2398 if (i < num_tc) {
2399 adapter->ch_config.ch_info[i].count =
2400 mqprio_qopt->qopt.count[i];
2401 adapter->ch_config.ch_info[i].offset =
2402 mqprio_qopt->qopt.offset[i];
2403 total_qps += mqprio_qopt->qopt.count[i];
2404 max_tx_rate = mqprio_qopt->max_rate[i];
2405 /* convert to Mbps */
2406 max_tx_rate = div_u64(max_tx_rate,
2407 I40EVF_MBPS_DIVISOR);
2408 adapter->ch_config.ch_info[i].max_tx_rate =
2409 max_tx_rate;
2410 } else {
2411 adapter->ch_config.ch_info[i].count = 1;
2412 adapter->ch_config.ch_info[i].offset = 0;
2413 }
2414 }
2415 adapter->ch_config.total_qps = total_qps;
2416 netif_tx_stop_all_queues(netdev);
2417 netif_tx_disable(netdev);
2418 adapter->aq_required |= I40EVF_FLAG_AQ_ENABLE_CHANNELS;
2419 netdev_reset_tc(netdev);
2420 /* Report the tc mapping up the stack */
2421 netdev_set_num_tc(adapter->netdev, num_tc);
2422 for (i = 0; i < I40EVF_MAX_TRAFFIC_CLASS; i++) {
2423 u16 qcount = mqprio_qopt->qopt.count[i];
2424 u16 qoffset = mqprio_qopt->qopt.offset[i];
2426 if (i < num_tc)
2427 netdev_set_tc_queue(netdev, netdev_tc++, qcount,
2428 qoffset);
2429 }
2430 }
2431 exit:
2432 return ret;
2433 }
2435 /**
2436 * i40evf_parse_cls_flower - Parse tc flower filters provided by kernel
2437 * @adapter: board private structure
2438 * @cls_flower: pointer to struct tc_cls_flower_offload
2439 * @filter: pointer to cloud filter structure
2440 */
2441 static int i40evf_parse_cls_flower(struct i40evf_adapter *adapter,
2442 struct tc_cls_flower_offload *f,
2443 struct i40evf_cloud_filter *filter)
2444 {
2445 u16 n_proto_mask = 0;
2446 u16 n_proto_key = 0;
2447 u8 field_flags = 0;
2448 u16 addr_type = 0;
2449 u16 n_proto = 0;
2450 int i = 0;
2451 struct virtchnl_filter *vf = &filter->f;
2453 if (f->dissector->used_keys &
2454 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
2455 BIT(FLOW_DISSECTOR_KEY_BASIC) |
2456 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
2457 BIT(FLOW_DISSECTOR_KEY_VLAN) |
2458 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
2459 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
2460 BIT(FLOW_DISSECTOR_KEY_PORTS) |
2461 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
2462 dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%x\n",
2463 f->dissector->used_keys);
2464 return -EOPNOTSUPP;
2465 }
2467 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
2468 struct flow_dissector_key_keyid *mask =
2469 skb_flow_dissector_target(f->dissector,
2470 FLOW_DISSECTOR_KEY_ENC_KEYID,
2471 f->mask);
2473 if (mask->keyid != 0)
2474 field_flags |= I40EVF_CLOUD_FIELD_TEN_ID;
2475 }
2477 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
2478 struct flow_dissector_key_basic *key =
2479 skb_flow_dissector_target(f->dissector,
2480 FLOW_DISSECTOR_KEY_BASIC,
2481 f->key);
2483 struct flow_dissector_key_basic *mask =
2484 skb_flow_dissector_target(f->dissector,
2485 FLOW_DISSECTOR_KEY_BASIC,
2486 f->mask);
2487 n_proto_key = ntohs(key->n_proto);
2488 n_proto_mask = ntohs(mask->n_proto);
2490 if (n_proto_key == ETH_P_ALL) {
2491 n_proto_key = 0;
2492 n_proto_mask = 0;
2493 }
2494 n_proto = n_proto_key & n_proto_mask;
2495 if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6)
2496 return -EINVAL;
2497 if (n_proto == ETH_P_IPV6) {
2498 /* specify flow type as TCP IPv6 */
2499 vf->flow_type = VIRTCHNL_TCP_V6_FLOW;
2500 }
2502 if (key->ip_proto != IPPROTO_TCP) {
2503 dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n");
2504 return -EINVAL;
2505 }
2506 }
2508 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
2509 struct flow_dissector_key_eth_addrs *key =
2510 skb_flow_dissector_target(f->dissector,
2511 FLOW_DISSECTOR_KEY_ETH_ADDRS,
2512 f->key);
2514 struct flow_dissector_key_eth_addrs *mask =
2515 skb_flow_dissector_target(f->dissector,
2516 FLOW_DISSECTOR_KEY_ETH_ADDRS,
2517 f->mask);
2518 /* use is_broadcast and is_zero to check for all 0xf or 0 */
2519 if (!is_zero_ether_addr(mask->dst)) {
2520 if (is_broadcast_ether_addr(mask->dst)) {
2521 field_flags |= I40EVF_CLOUD_FIELD_OMAC;
2522 } else {
2523 dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n",
2524 mask->dst);
2525 return I40E_ERR_CONFIG;
2526 }
2527 }
2529 if (!is_zero_ether_addr(mask->src)) {
2530 if (is_broadcast_ether_addr(mask->src)) {
2531 field_flags |= I40EVF_CLOUD_FIELD_IMAC;
2532 } else {
2533 dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n",
2534 mask->src);
2535 return I40E_ERR_CONFIG;
2536 }
2537 }
2539 if (!is_zero_ether_addr(key->dst))
2540 if (is_valid_ether_addr(key->dst) ||
2541 is_multicast_ether_addr(key->dst)) {
2542 /* set the mask if a valid dst_mac address */
2543 for (i = 0; i < ETH_ALEN; i++)
2544 vf->mask.tcp_spec.dst_mac[i] |= 0xff;
2545 ether_addr_copy(vf->data.tcp_spec.dst_mac,
2546 key->dst);
2547 }
2549 if (!is_zero_ether_addr(key->src))
2550 if (is_valid_ether_addr(key->src) ||
2551 is_multicast_ether_addr(key->src)) {
2552 /* set the mask if a valid dst_mac address */
2553 for (i = 0; i < ETH_ALEN; i++)
2554 vf->mask.tcp_spec.src_mac[i] |= 0xff;
2555 ether_addr_copy(vf->data.tcp_spec.src_mac,
2556 key->src);
2557 }
2558 }
2560 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) {
2561 struct flow_dissector_key_vlan *key =
2562 skb_flow_dissector_target(f->dissector,
2563 FLOW_DISSECTOR_KEY_VLAN,
2564 f->key);
2565 struct flow_dissector_key_vlan *mask =
2566 skb_flow_dissector_target(f->dissector,
2567 FLOW_DISSECTOR_KEY_VLAN,
2568 f->mask);
2570 if (mask->vlan_id) {
2571 if (mask->vlan_id == VLAN_VID_MASK) {
2572 field_flags |= I40EVF_CLOUD_FIELD_IVLAN;
2573 } else {
2574 dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n",
2575 mask->vlan_id);
2576 return I40E_ERR_CONFIG;
2577 }
2578 }
2579 vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff);
2580 vf->data.tcp_spec.vlan_id = cpu_to_be16(key->vlan_id);
2581 }
2583 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
2584 struct flow_dissector_key_control *key =
2585 skb_flow_dissector_target(f->dissector,
2586 FLOW_DISSECTOR_KEY_CONTROL,
2587 f->key);
2589 addr_type = key->addr_type;
2590 }
2592 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
2593 struct flow_dissector_key_ipv4_addrs *key =
2594 skb_flow_dissector_target(f->dissector,
2595 FLOW_DISSECTOR_KEY_IPV4_ADDRS,
2596 f->key);
2597 struct flow_dissector_key_ipv4_addrs *mask =
2598 skb_flow_dissector_target(f->dissector,
2599 FLOW_DISSECTOR_KEY_IPV4_ADDRS,
2600 f->mask);
2602 if (mask->dst) {
2603 if (mask->dst == cpu_to_be32(0xffffffff)) {
2604 field_flags |= I40EVF_CLOUD_FIELD_IIP;
2605 } else {
2606 dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n",
2607 be32_to_cpu(mask->dst));
2608 return I40E_ERR_CONFIG;
2609 }
2610 }
2612 if (mask->src) {
2613 if (mask->src == cpu_to_be32(0xffffffff)) {
2614 field_flags |= I40EVF_CLOUD_FIELD_IIP;
2615 } else {
2616 dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n",
2617 be32_to_cpu(mask->dst));
2618 return I40E_ERR_CONFIG;
2619 }
2620 }
2622 if (field_flags & I40EVF_CLOUD_FIELD_TEN_ID) {
2623 dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n");
2624 return I40E_ERR_CONFIG;
2625 }
2626 if (key->dst) {
2627 vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff);
2628 vf->data.tcp_spec.dst_ip[0] = key->dst;
2629 }
2630 if (key->src) {
2631 vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff);
2632 vf->data.tcp_spec.src_ip[0] = key->src;
2633 }
2634 }
2636 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
2637 struct flow_dissector_key_ipv6_addrs *key =
2638 skb_flow_dissector_target(f->dissector,
2639 FLOW_DISSECTOR_KEY_IPV6_ADDRS,
2640 f->key);
2641 struct flow_dissector_key_ipv6_addrs *mask =
2642 skb_flow_dissector_target(f->dissector,
2643 FLOW_DISSECTOR_KEY_IPV6_ADDRS,
2644 f->mask);
2646 /* validate mask, make sure it is not IPV6_ADDR_ANY */
2647 if (ipv6_addr_any(&mask->dst)) {
2648 dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n",
2649 IPV6_ADDR_ANY);
2650 return I40E_ERR_CONFIG;
2651 }
2653 /* src and dest IPv6 address should not be LOOPBACK
2654 * (0:0:0:0:0:0:0:1) which can be represented as ::1
2655 */
2656 if (ipv6_addr_loopback(&key->dst) ||
2657 ipv6_addr_loopback(&key->src)) {
2658 dev_err(&adapter->pdev->dev,
2659 "ipv6 addr should not be loopback\n");
2660 return I40E_ERR_CONFIG;
2661 }
2662 if (!ipv6_addr_any(&mask->dst) || !ipv6_addr_any(&mask->src))
2663 field_flags |= I40EVF_CLOUD_FIELD_IIP;
2665 for (i = 0; i < 4; i++)
2666 vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff);
2667 memcpy(&vf->data.tcp_spec.dst_ip, &key->dst.s6_addr32,
2668 sizeof(vf->data.tcp_spec.dst_ip));
2669 for (i = 0; i < 4; i++)
2670 vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff);
2671 memcpy(&vf->data.tcp_spec.src_ip, &key->src.s6_addr32,
2672 sizeof(vf->data.tcp_spec.src_ip));
2673 }
2674 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_PORTS)) {
2675 struct flow_dissector_key_ports *key =
2676 skb_flow_dissector_target(f->dissector,
2677 FLOW_DISSECTOR_KEY_PORTS,
2678 f->key);
2679 struct flow_dissector_key_ports *mask =
2680 skb_flow_dissector_target(f->dissector,
2681 FLOW_DISSECTOR_KEY_PORTS,
2682 f->mask);
2684 if (mask->src) {
2685 if (mask->src == cpu_to_be16(0xffff)) {
2686 field_flags |= I40EVF_CLOUD_FIELD_IIP;
2687 } else {
2688 dev_err(&adapter->pdev->dev, "Bad src port mask %u\n",
2689 be16_to_cpu(mask->src));
2690 return I40E_ERR_CONFIG;
2691 }
2692 }
2694 if (mask->dst) {
2695 if (mask->dst == cpu_to_be16(0xffff)) {
2696 field_flags |= I40EVF_CLOUD_FIELD_IIP;
2697 } else {
2698 dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n",
2699 be16_to_cpu(mask->dst));
2700 return I40E_ERR_CONFIG;
2701 }
2702 }
2703 if (key->dst) {
2704 vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff);
2705 vf->data.tcp_spec.dst_port = key->dst;
2706 }
2708 if (key->src) {
2709 vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff);
2710 vf->data.tcp_spec.src_port = key->src;
2711 }
2712 }
2713 vf->field_flags = field_flags;
2715 return 0;
2716 }
2718 /**
2719 * i40evf_handle_tclass - Forward to a traffic class on the device
2720 * @adapter: board private structure
2721 * @tc: traffic class index on the device
2722 * @filter: pointer to cloud filter structure
2723 */
2724 static int i40evf_handle_tclass(struct i40evf_adapter *adapter, u32 tc,
2725 struct i40evf_cloud_filter *filter)
2726 {
2727 if (tc == 0)
2728 return 0;
2729 if (tc < adapter->num_tc) {
2730 if (!filter->f.data.tcp_spec.dst_port) {
2731 dev_err(&adapter->pdev->dev,
2732 "Specify destination port to redirect to traffic class other than TC0\n");
2733 return -EINVAL;
2734 }
2735 }
2736 /* redirect to a traffic class on the same device */
2737 filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT;
2738 filter->f.action_meta = tc;
2739 return 0;
2740 }
2742 /**
2743 * i40evf_configure_clsflower - Add tc flower filters
2744 * @adapter: board private structure
2745 * @cls_flower: Pointer to struct tc_cls_flower_offload
2746 */
2747 static int i40evf_configure_clsflower(struct i40evf_adapter *adapter,
2748 struct tc_cls_flower_offload *cls_flower)
2749 {
2750 int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
2751 struct i40evf_cloud_filter *filter = NULL;
2752 int err = -EINVAL, count = 50;
2754 if (tc < 0) {
2755 dev_err(&adapter->pdev->dev, "Invalid traffic class\n");
2756 return -EINVAL;
2757 }
2759 filter = kzalloc(sizeof(*filter), GFP_KERNEL);
2760 if (!filter)
2761 return -ENOMEM;
2763 while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
2764 &adapter->crit_section)) {
2765 if (--count == 0)
2766 goto err;
2767 udelay(1);
2768 }
2770 filter->cookie = cls_flower->cookie;
2772 /* set the mask to all zeroes to begin with */
2773 memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec));
2774 /* start out with flow type and eth type IPv4 to begin with */
2775 filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW;
2776 err = i40evf_parse_cls_flower(adapter, cls_flower, filter);
2777 if (err < 0)
2778 goto err;
2780 err = i40evf_handle_tclass(adapter, tc, filter);
2781 if (err < 0)
2782 goto err;
2784 /* add filter to the list */
2785 spin_lock_bh(&adapter->cloud_filter_list_lock);
2786 list_add_tail(&filter->list, &adapter->cloud_filter_list);
2787 adapter->num_cloud_filters++;
2788 filter->add = true;
2789 adapter->aq_required |= I40EVF_FLAG_AQ_ADD_CLOUD_FILTER;
2790 spin_unlock_bh(&adapter->cloud_filter_list_lock);
2791 err:
2792 if (err)
2793 kfree(filter);
2795 clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
2796 return err;
2797 }
2799 /* i40evf_find_cf - Find the cloud filter in the list
2800 * @adapter: Board private structure
2801 * @cookie: filter specific cookie
2802 *
2803 * Returns ptr to the filter object or NULL. Must be called while holding the
2804 * cloud_filter_list_lock.
2805 */
2806 static struct i40evf_cloud_filter *i40evf_find_cf(struct i40evf_adapter *adapter,
2807 unsigned long *cookie)
2808 {
2809 struct i40evf_cloud_filter *filter = NULL;
2811 if (!cookie)
2812 return NULL;
2814 list_for_each_entry(filter, &adapter->cloud_filter_list, list) {
2815 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
2816 return filter;
2817 }
2818 return NULL;
2819 }
2821 /**
2822 * i40evf_delete_clsflower - Remove tc flower filters
2823 * @adapter: board private structure
2824 * @cls_flower: Pointer to struct tc_cls_flower_offload
2825 */
2826 static int i40evf_delete_clsflower(struct i40evf_adapter *adapter,
2827 struct tc_cls_flower_offload *cls_flower)
2828 {
2829 struct i40evf_cloud_filter *filter = NULL;
2830 int err = 0;
2832 spin_lock_bh(&adapter->cloud_filter_list_lock);
2833 filter = i40evf_find_cf(adapter, &cls_flower->cookie);
2834 if (filter) {
2835 filter->del = true;
2836 adapter->aq_required |= I40EVF_FLAG_AQ_DEL_CLOUD_FILTER;
2837 } else {
2838 err = -EINVAL;
2839 }
2840 spin_unlock_bh(&adapter->cloud_filter_list_lock);
2842 return err;
2843 }
2845 /**
2846 * i40evf_setup_tc_cls_flower - flower classifier offloads
2847 * @netdev: net device to configure
2848 * @type_data: offload data
2849 */
2850 static int i40evf_setup_tc_cls_flower(struct i40evf_adapter *adapter,
2851 struct tc_cls_flower_offload *cls_flower)
2852 {
2853 if (cls_flower->common.chain_index)
2854 return -EOPNOTSUPP;
2856 switch (cls_flower->command) {
2857 case TC_CLSFLOWER_REPLACE:
2858 return i40evf_configure_clsflower(adapter, cls_flower);
2859 case TC_CLSFLOWER_DESTROY:
2860 return i40evf_delete_clsflower(adapter, cls_flower);
2861 case TC_CLSFLOWER_STATS:
2862 return -EOPNOTSUPP;
2863 default:
2864 return -EOPNOTSUPP;
2865 }
2866 }
2868 /**
2869 * i40evf_setup_tc_block_cb - block callback for tc
2870 * @type: type of offload
2871 * @type_data: offload data
2872 * @cb_priv:
2873 *
2874 * This function is the block callback for traffic classes
2875 **/
2876 static int i40evf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
2877 void *cb_priv)
2878 {
2879 switch (type) {
2880 case TC_SETUP_CLSFLOWER:
2881 return i40evf_setup_tc_cls_flower(cb_priv, type_data);
2882 default:
2883 return -EOPNOTSUPP;
2884 }
2885 }
2887 /**
2888 * i40evf_setup_tc_block - register callbacks for tc
2889 * @netdev: network interface device structure
2890 * @f: tc offload data
2891 *
2892 * This function registers block callbacks for tc
2893 * offloads
2894 **/
2895 static int i40evf_setup_tc_block(struct net_device *dev,
2896 struct tc_block_offload *f)
2897 {
2898 struct i40evf_adapter *adapter = netdev_priv(dev);
2900 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
2901 return -EOPNOTSUPP;
2903 switch (f->command) {
2904 case TC_BLOCK_BIND:
2905 return tcf_block_cb_register(f->block, i40evf_setup_tc_block_cb,
2906 adapter, adapter, f->extack);
2907 case TC_BLOCK_UNBIND:
2908 tcf_block_cb_unregister(f->block, i40evf_setup_tc_block_cb,
2909 adapter);
2910 return 0;
2911 default:
2912 return -EOPNOTSUPP;
2913 }
2914 }
2916 /**
2917 * i40evf_setup_tc - configure multiple traffic classes
2918 * @netdev: network interface device structure
2919 * @type: type of offload
2920 * @type_date: tc offload data
2921 *
2922 * This function is the callback to ndo_setup_tc in the
2923 * netdev_ops.
2924 *
2925 * Returns 0 on success
2926 **/
2927 static int i40evf_setup_tc(struct net_device *netdev, enum tc_setup_type type,
2928 void *type_data)
2929 {
2930 switch (type) {
2931 case TC_SETUP_QDISC_MQPRIO:
2932 return __i40evf_setup_tc(netdev, type_data);
2933 case TC_SETUP_BLOCK:
2934 return i40evf_setup_tc_block(netdev, type_data);
2935 default:
2936 return -EOPNOTSUPP;
2937 }
2938 }
2940 /**
2941 * i40evf_open - Called when a network interface is made active
2942 * @netdev: network interface device structure
2943 *
2944 * Returns 0 on success, negative value on failure
2945 *
2946 * The open entry point is called when a network interface is made
2947 * active by the system (IFF_UP). At this point all resources needed
2948 * for transmit and receive operations are allocated, the interrupt
2949 * handler is registered with the OS, the watchdog timer is started,
2950 * and the stack is notified that the interface is ready.
2951 **/
2952 static int i40evf_open(struct net_device *netdev)
2953 {
2954 struct i40evf_adapter *adapter = netdev_priv(netdev);
2955 int err;
2957 if (adapter->flags & I40EVF_FLAG_PF_COMMS_FAILED) {
2958 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n");
2959 return -EIO;
2960 }
2962 while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
2963 &adapter->crit_section))
2964 usleep_range(500, 1000);
2966 if (adapter->state != __I40EVF_DOWN) {
2967 err = -EBUSY;
2968 goto err_unlock;
2969 }
2971 /* allocate transmit descriptors */
2972 err = i40evf_setup_all_tx_resources(adapter);
2973 if (err)
2974 goto err_setup_tx;
2976 /* allocate receive descriptors */
2977 err = i40evf_setup_all_rx_resources(adapter);
2978 if (err)
2979 goto err_setup_rx;
2981 /* clear any pending interrupts, may auto mask */
2982 err = i40evf_request_traffic_irqs(adapter, netdev->name);
2983 if (err)
2984 goto err_req_irq;
2986 spin_lock_bh(&adapter->mac_vlan_list_lock);
2988 i40evf_add_filter(adapter, adapter->hw.mac.addr);
2990 spin_unlock_bh(&adapter->mac_vlan_list_lock);
2992 i40evf_configure(adapter);
2994 i40evf_up_complete(adapter);
2996 i40evf_irq_enable(adapter, true);
2998 clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
3000 return 0;
3002 err_req_irq:
3003 i40evf_down(adapter);
3004 i40evf_free_traffic_irqs(adapter);
3005 err_setup_rx:
3006 i40evf_free_all_rx_resources(adapter);
3007 err_setup_tx:
3008 i40evf_free_all_tx_resources(adapter);
3009 err_unlock:
3010 clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
3012 return err;
3013 }
3015 /**
3016 * i40evf_close - Disables a network interface
3017 * @netdev: network interface device structure
3018 *
3019 * Returns 0, this is not allowed to fail
3020 *
3021 * The close entry point is called when an interface is de-activated
3022 * by the OS. The hardware is still under the drivers control, but
3023 * needs to be disabled. All IRQs except vector 0 (reserved for admin queue)
3024 * are freed, along with all transmit and receive resources.
3025 **/
3026 static int i40evf_close(struct net_device *netdev)
3027 {
3028 struct i40evf_adapter *adapter = netdev_priv(netdev);
3029 int status;
3031 if (adapter->state <= __I40EVF_DOWN_PENDING)
3032 return 0;
3034 while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
3035 &adapter->crit_section))
3036 usleep_range(500, 1000);
3038 set_bit(__I40E_VSI_DOWN, adapter->vsi.state);
3039 if (CLIENT_ENABLED(adapter))
3040 adapter->flags |= I40EVF_FLAG_CLIENT_NEEDS_CLOSE;
3042 i40evf_down(adapter);
3043 adapter->state = __I40EVF_DOWN_PENDING;
3044 i40evf_free_traffic_irqs(adapter);
3046 clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
3048 /* We explicitly don't free resources here because the hardware is
3049 * still active and can DMA into memory. Resources are cleared in
3050 * i40evf_virtchnl_completion() after we get confirmation from the PF
3051 * driver that the rings have been stopped.
3052 *
3053 * Also, we wait for state to transition to __I40EVF_DOWN before
3054 * returning. State change occurs in i40evf_virtchnl_completion() after
3055 * VF resources are released (which occurs after PF driver processes and
3056 * responds to admin queue commands).
3057 */
3059 status = wait_event_timeout(adapter->down_waitqueue,
3060 adapter->state == __I40EVF_DOWN,
3061 msecs_to_jiffies(200));
3062 if (!status)
3063 netdev_warn(netdev, "Device resources not yet released\n");
3064 return 0;
3065 }
3067 /**
3068 * i40evf_change_mtu - Change the Maximum Transfer Unit
3069 * @netdev: network interface device structure
3070 * @new_mtu: new value for maximum frame size
3071 *
3072 * Returns 0 on success, negative on failure
3073 **/
3074 static int i40evf_change_mtu(struct net_device *netdev, int new_mtu)
3075 {
3076 struct i40evf_adapter *adapter = netdev_priv(netdev);
3078 netdev->mtu = new_mtu;
3079 if (CLIENT_ENABLED(adapter)) {
3080 i40evf_notify_client_l2_params(&adapter->vsi);
3081 adapter->flags |= I40EVF_FLAG_SERVICE_CLIENT_REQUESTED;
3082 }
3083 adapter->flags |= I40EVF_FLAG_RESET_NEEDED;
3084 schedule_work(&adapter->reset_task);
3086 return 0;
3087 }
3089 /**
3090 * i40e_set_features - set the netdev feature flags
3091 * @netdev: ptr to the netdev being adjusted
3092 * @features: the feature set that the stack is suggesting
3093 * Note: expects to be called while under rtnl_lock()
3094 **/
3095 static int i40evf_set_features(struct net_device *netdev,
3096 netdev_features_t features)
3097 {
3098 struct i40evf_adapter *adapter = netdev_priv(netdev);
3100 /* Don't allow changing VLAN_RX flag when VLAN is set for VF
3101 * and return an error in this case
3102 */
3103 if (VLAN_ALLOWED(adapter)) {
3104 if (features & NETIF_F_HW_VLAN_CTAG_RX)
3105 adapter->aq_required |=
3106 I40EVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
3107 else
3108 adapter->aq_required |=
3109 I40EVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
3110 } else if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX) {
3111 return -EINVAL;
3112 }
3114 return 0;
3115 }
3117 /**
3118 * i40evf_features_check - Validate encapsulated packet conforms to limits
3119 * @skb: skb buff
3120 * @dev: This physical port's netdev
3121 * @features: Offload features that the stack believes apply
3122 **/
3123 static netdev_features_t i40evf_features_check(struct sk_buff *skb,
3124 struct net_device *dev,
3125 netdev_features_t features)
3126 {
3127 size_t len;
3129 /* No point in doing any of this if neither checksum nor GSO are
3130 * being requested for this frame. We can rule out both by just
3131 * checking for CHECKSUM_PARTIAL
3132 */
3133 if (skb->ip_summed != CHECKSUM_PARTIAL)
3134 return features;
3136 /* We cannot support GSO if the MSS is going to be less than
3137 * 64 bytes. If it is then we need to drop support for GSO.
3138 */
3139 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
3140 features &= ~NETIF_F_GSO_MASK;
3142 /* MACLEN can support at most 63 words */
3143 len = skb_network_header(skb) - skb->data;
3144 if (len & ~(63 * 2))
3145 goto out_err;
3147 /* IPLEN and EIPLEN can support at most 127 dwords */
3148 len = skb_transport_header(skb) - skb_network_header(skb);
3149 if (len & ~(127 * 4))
3150 goto out_err;
3152 if (skb->encapsulation) {
3153 /* L4TUNLEN can support 127 words */
3154 len = skb_inner_network_header(skb) - skb_transport_header(skb);
3155 if (len & ~(127 * 2))
3156 goto out_err;
3158 /* IPLEN can support at most 127 dwords */
3159 len = skb_inner_transport_header(skb) -
3160 skb_inner_network_header(skb);
3161 if (len & ~(127 * 4))
3162 goto out_err;
3163 }
3165 /* No need to validate L4LEN as TCP is the only protocol with a
3166 * a flexible value and we support all possible values supported
3167 * by TCP, which is at most 15 dwords
3168 */
3170 return features;
3171 out_err:
3172 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3173 }
3175 /**
3176 * i40evf_fix_features - fix up the netdev feature bits
3177 * @netdev: our net device
3178 * @features: desired feature bits
3179 *
3180 * Returns fixed-up features bits
3181 **/
3182 static netdev_features_t i40evf_fix_features(struct net_device *netdev,
3183 netdev_features_t features)
3184 {
3185 struct i40evf_adapter *adapter = netdev_priv(netdev);
3187 if (!(adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN))
3188 features &= ~(NETIF_F_HW_VLAN_CTAG_TX |
3189 NETIF_F_HW_VLAN_CTAG_RX |
3190 NETIF_F_HW_VLAN_CTAG_FILTER);
3192 return features;
3193 }
3195 static const struct net_device_ops i40evf_netdev_ops = {
3196 .ndo_open = i40evf_open,
3197 .ndo_stop = i40evf_close,
3198 .ndo_start_xmit = i40evf_xmit_frame,
3199 .ndo_set_rx_mode = i40evf_set_rx_mode,
3200 .ndo_validate_addr = eth_validate_addr,
3201 .ndo_set_mac_address = i40evf_set_mac,
3202 .ndo_change_mtu = i40evf_change_mtu,
3203 .ndo_tx_timeout = i40evf_tx_timeout,
3204 .ndo_vlan_rx_add_vid = i40evf_vlan_rx_add_vid,
3205 .ndo_vlan_rx_kill_vid = i40evf_vlan_rx_kill_vid,
3206 .ndo_features_check = i40evf_features_check,
3207 .ndo_fix_features = i40evf_fix_features,
3208 .ndo_set_features = i40evf_set_features,
3209 .ndo_setup_tc = i40evf_setup_tc,
3210 };
3212 /**
3213 * i40evf_check_reset_complete - check that VF reset is complete
3214 * @hw: pointer to hw struct
3215 *
3216 * Returns 0 if device is ready to use, or -EBUSY if it's in reset.
3217 **/
3218 static int i40evf_check_reset_complete(struct i40e_hw *hw)
3219 {
3220 u32 rstat;
3221 int i;
3223 for (i = 0; i < 100; i++) {
3224 rstat = rd32(hw, I40E_VFGEN_RSTAT) &
3225 I40E_VFGEN_RSTAT_VFR_STATE_MASK;
3226 if ((rstat == VIRTCHNL_VFR_VFACTIVE) ||
3227 (rstat == VIRTCHNL_VFR_COMPLETED))
3228 return 0;
3229 usleep_range(10, 20);
3230 }
3231 return -EBUSY;
3232 }
3234 /**
3235 * i40evf_process_config - Process the config information we got from the PF
3236 * @adapter: board private structure
3237 *
3238 * Verify that we have a valid config struct, and set up our netdev features
3239 * and our VSI struct.
3240 **/
3241 int i40evf_process_config(struct i40evf_adapter *adapter)
3242 {
3243 struct virtchnl_vf_resource *vfres = adapter->vf_res;
3244 int i, num_req_queues = adapter->num_req_queues;
3245 struct net_device *netdev = adapter->netdev;
3246 struct i40e_vsi *vsi = &adapter->vsi;
3247 netdev_features_t hw_enc_features;
3248 netdev_features_t hw_features;
3250 /* got VF config message back from PF, now we can parse it */
3251 for (i = 0; i < vfres->num_vsis; i++) {
3252 if (vfres->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV)
3253 adapter->vsi_res = &vfres->vsi_res[i];
3254 }
3255 if (!adapter->vsi_res) {
3256 dev_err(&adapter->pdev->dev, "No LAN VSI found\n");
3257 return -ENODEV;
3258 }
3260 if (num_req_queues &&
3261 num_req_queues != adapter->vsi_res->num_queue_pairs) {
3262 /* Problem. The PF gave us fewer queues than what we had
3263 * negotiated in our request. Need a reset to see if we can't
3264 * get back to a working state.
3265 */
3266 dev_err(&adapter->pdev->dev,
3267 "Requested %d queues, but PF only gave us %d.\n",
3268 num_req_queues,
3269 adapter->vsi_res->num_queue_pairs);
3270 adapter->flags |= I40EVF_FLAG_REINIT_ITR_NEEDED;
3271 adapter->num_req_queues = adapter->vsi_res->num_queue_pairs;
3272 i40evf_schedule_reset(adapter);
3273 return -ENODEV;
3274 }
3275 adapter->num_req_queues = 0;
3277 hw_enc_features = NETIF_F_SG |
3278 NETIF_F_IP_CSUM |
3279 NETIF_F_IPV6_CSUM |
3280 NETIF_F_HIGHDMA |
3281 NETIF_F_SOFT_FEATURES |
3282 NETIF_F_TSO |
3283 NETIF_F_TSO_ECN |
3284 NETIF_F_TSO6 |
3285 NETIF_F_SCTP_CRC |
3286 NETIF_F_RXHASH |
3287 NETIF_F_RXCSUM |
3288 0;
3290 /* advertise to stack only if offloads for encapsulated packets is
3291 * supported
3292 */
3293 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) {
3294 hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL |
3295 NETIF_F_GSO_GRE |
3296 NETIF_F_GSO_GRE_CSUM |
3297 NETIF_F_GSO_IPXIP4 |
3298 NETIF_F_GSO_IPXIP6 |
3299 NETIF_F_GSO_UDP_TUNNEL_CSUM |
3300 NETIF_F_GSO_PARTIAL |
3301 0;
3303 if (!(vfres->vf_cap_flags &
3304 VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))
3305 netdev->gso_partial_features |=
3306 NETIF_F_GSO_UDP_TUNNEL_CSUM;
3308 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
3309 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
3310 netdev->hw_enc_features |= hw_enc_features;
3311 }
3312 /* record features VLANs can make use of */
3313 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
3315 /* Write features and hw_features separately to avoid polluting
3316 * with, or dropping, features that are set when we registered.
3317 */
3318 hw_features = hw_enc_features;
3320 /* Enable VLAN features if supported */
3321 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
3322 hw_features |= (NETIF_F_HW_VLAN_CTAG_TX |
3323 NETIF_F_HW_VLAN_CTAG_RX);
3324 /* Enable cloud filter if ADQ is supported */
3325 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)
3326 hw_features |= NETIF_F_HW_TC;
3328 netdev->hw_features |= hw_features;
3330 netdev->features |= hw_features;
3332 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
3333 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
3335 /* Do not turn on offloads when they are requested to be turned off.
3336 * TSO needs minimum 576 bytes to work correctly.
3337 */
3338 if (netdev->wanted_features) {
3339 if (!(netdev->wanted_features & NETIF_F_TSO) ||
3340 netdev->mtu < 576)
3341 netdev->features &= ~NETIF_F_TSO;
3342 if (!(netdev->wanted_features & NETIF_F_TSO6) ||
3343 netdev->mtu < 576)
3344 netdev->features &= ~NETIF_F_TSO6;
3345 if (!(netdev->wanted_features & NETIF_F_TSO_ECN))
3346 netdev->features &= ~NETIF_F_TSO_ECN;
3347 if (!(netdev->wanted_features & NETIF_F_GRO))
3348 netdev->features &= ~NETIF_F_GRO;
3349 if (!(netdev->wanted_features & NETIF_F_GSO))
3350 netdev->features &= ~NETIF_F_GSO;
3351 }
3353 adapter->vsi.id = adapter->vsi_res->vsi_id;
3355 adapter->vsi.back = adapter;
3356 adapter->vsi.base_vector = 1;
3357 adapter->vsi.work_limit = I40E_DEFAULT_IRQ_WORK;
3358 vsi->netdev = adapter->netdev;
3359 vsi->qs_handle = adapter->vsi_res->qset_handle;
3360 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
3361 adapter->rss_key_size = vfres->rss_key_size;
3362 adapter->rss_lut_size = vfres->rss_lut_size;
3363 } else {
3364 adapter->rss_key_size = I40EVF_HKEY_ARRAY_SIZE;
3365 adapter->rss_lut_size = I40EVF_HLUT_ARRAY_SIZE;
3366 }
3368 return 0;
3369 }
3371 /**
3372 * i40evf_init_task - worker thread to perform delayed initialization
3373 * @work: pointer to work_struct containing our data
3374 *
3375 * This task completes the work that was begun in probe. Due to the nature
3376 * of VF-PF communications, we may need to wait tens of milliseconds to get
3377 * responses back from the PF. Rather than busy-wait in probe and bog down the
3378 * whole system, we'll do it in a task so we can sleep.
3379 * This task only runs during driver init. Once we've established
3380 * communications with the PF driver and set up our netdev, the watchdog
3381 * takes over.
3382 **/
3383 static void i40evf_init_task(struct work_struct *work)
3384 {
3385 struct i40evf_adapter *adapter = container_of(work,
3386 struct i40evf_adapter,
3387 init_task.work);
3388 struct net_device *netdev = adapter->netdev;
3389 struct i40e_hw *hw = &adapter->hw;
3390 struct pci_dev *pdev = adapter->pdev;
3391 int err, bufsz;
3393 switch (adapter->state) {
3394 case __I40EVF_STARTUP:
3395 /* driver loaded, probe complete */
3396 adapter->flags &= ~I40EVF_FLAG_PF_COMMS_FAILED;
3397 adapter->flags &= ~I40EVF_FLAG_RESET_PENDING;
3398 err = i40e_set_mac_type(hw);
3399 if (err) {
3400 dev_err(&pdev->dev, "Failed to set MAC type (%d)\n",
3401 err);
3402 goto err;
3403 }
3404 err = i40evf_check_reset_complete(hw);
3405 if (err) {
3406 dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n",
3407 err);
3408 goto err;
3409 }
3410 hw->aq.num_arq_entries = I40EVF_AQ_LEN;
3411 hw->aq.num_asq_entries = I40EVF_AQ_LEN;
3412 hw->aq.arq_buf_size = I40EVF_MAX_AQ_BUF_SIZE;
3413 hw->aq.asq_buf_size = I40EVF_MAX_AQ_BUF_SIZE;
3415 err = i40evf_init_adminq(hw);
3416 if (err) {
3417 dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n",
3418 err);
3419 goto err;
3420 }
3421 err = i40evf_send_api_ver(adapter);
3422 if (err) {
3423 dev_err(&pdev->dev, "Unable to send to PF (%d)\n", err);
3424 i40evf_shutdown_adminq(hw);
3425 goto err;
3426 }
3427 adapter->state = __I40EVF_INIT_VERSION_CHECK;
3428 goto restart;
3429 case __I40EVF_INIT_VERSION_CHECK:
3430 if (!i40evf_asq_done(hw)) {
3431 dev_err(&pdev->dev, "Admin queue command never completed\n");
3432 i40evf_shutdown_adminq(hw);
3433 adapter->state = __I40EVF_STARTUP;
3434 goto err;
3435 }
3437 /* aq msg sent, awaiting reply */
3438 err = i40evf_verify_api_ver(adapter);
3439 if (err) {
3440 if (err == I40E_ERR_ADMIN_QUEUE_NO_WORK)
3441 err = i40evf_send_api_ver(adapter);
3442 else
3443 dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n",
3444 adapter->pf_version.major,
3445 adapter->pf_version.minor,
3446 VIRTCHNL_VERSION_MAJOR,
3447 VIRTCHNL_VERSION_MINOR);
3448 goto err;
3449 }
3450 err = i40evf_send_vf_config_msg(adapter);
3451 if (err) {
3452 dev_err(&pdev->dev, "Unable to send config request (%d)\n",
3453 err);
3454 goto err;
3455 }
3456 adapter->state = __I40EVF_INIT_GET_RESOURCES;
3457 goto restart;
3458 case __I40EVF_INIT_GET_RESOURCES:
3459 /* aq msg sent, awaiting reply */
3460 if (!adapter->vf_res) {
3461 bufsz = sizeof(struct virtchnl_vf_resource) +
3462 (I40E_MAX_VF_VSI *
3463 sizeof(struct virtchnl_vsi_resource));
3464 adapter->vf_res = kzalloc(bufsz, GFP_KERNEL);
3465 if (!adapter->vf_res)
3466 goto err;
3467 }
3468 err = i40evf_get_vf_config(adapter);
3469 if (err == I40E_ERR_ADMIN_QUEUE_NO_WORK) {
3470 err = i40evf_send_vf_config_msg(adapter);
3471 goto err;
3472 } else if (err == I40E_ERR_PARAM) {
3473 /* We only get ERR_PARAM if the device is in a very bad
3474 * state or if we've been disabled for previous bad
3475 * behavior. Either way, we're done now.
3476 */
3477 i40evf_shutdown_adminq(hw);
3478 dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n");
3479 return;
3480 }
3481 if (err) {
3482 dev_err(&pdev->dev, "Unable to get VF config (%d)\n",
3483 err);
3484 goto err_alloc;
3485 }
3486 adapter->state = __I40EVF_INIT_SW;
3487 break;
3488 default:
3489 goto err_alloc;
3490 }
3492 if (i40evf_process_config(adapter))
3493 goto err_alloc;
3494 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
3496 adapter->flags |= I40EVF_FLAG_RX_CSUM_ENABLED;
3498 netdev->netdev_ops = &i40evf_netdev_ops;
3499 i40evf_set_ethtool_ops(netdev);
3500 netdev->watchdog_timeo = 5 * HZ;
3502 /* MTU range: 68 - 9710 */
3503 netdev->min_mtu = ETH_MIN_MTU;
3504 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
3506 if (!is_valid_ether_addr(adapter->hw.mac.addr)) {
3507 dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n",
3508 adapter->hw.mac.addr);
3509 eth_hw_addr_random(netdev);
3510 ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr);
3511 } else {
3512 adapter->flags |= I40EVF_FLAG_ADDR_SET_BY_PF;
3513 ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr);
3514 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr);
3515 }
3517 timer_setup(&adapter->watchdog_timer, i40evf_watchdog_timer, 0);
3518 mod_timer(&adapter->watchdog_timer, jiffies + 1);
3520 adapter->tx_desc_count = I40EVF_DEFAULT_TXD;
3521 adapter->rx_desc_count = I40EVF_DEFAULT_RXD;
3522 err = i40evf_init_interrupt_scheme(adapter);
3523 if (err)
3524 goto err_sw_init;
3525 i40evf_map_rings_to_vectors(adapter);
3526 if (adapter->vf_res->vf_cap_flags &
3527 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
3528 adapter->flags |= I40EVF_FLAG_WB_ON_ITR_CAPABLE;
3530 err = i40evf_request_misc_irq(adapter);
3531 if (err)
3532 goto err_sw_init;
3534 netif_carrier_off(netdev);
3535 adapter->link_up = false;
3537 if (!adapter->netdev_registered) {
3538 err = register_netdev(netdev);
3539 if (err)
3540 goto err_register;
3541 }
3543 adapter->netdev_registered = true;
3545 netif_tx_stop_all_queues(netdev);
3546 if (CLIENT_ALLOWED(adapter)) {
3547 err = i40evf_lan_add_device(adapter);
3548 if (err)
3549 dev_info(&pdev->dev, "Failed to add VF to client API service list: %d\n",
3550 err);
3551 }
3553 dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr);
3554 if (netdev->features & NETIF_F_GRO)
3555 dev_info(&pdev->dev, "GRO is enabled\n");
3557 adapter->state = __I40EVF_DOWN;
3558 set_bit(__I40E_VSI_DOWN, adapter->vsi.state);
3559 i40evf_misc_irq_enable(adapter);
3560 wake_up(&adapter->down_waitqueue);
3562 adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL);
3563 adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL);
3564 if (!adapter->rss_key || !adapter->rss_lut)
3565 goto err_mem;
3567 if (RSS_AQ(adapter)) {
3568 adapter->aq_required |= I40EVF_FLAG_AQ_CONFIGURE_RSS;
3569 mod_timer_pending(&adapter->watchdog_timer, jiffies + 1);
3570 } else {
3571 i40evf_init_rss(adapter);
3572 }
3573 return;
3574 restart:
3575 schedule_delayed_work(&adapter->init_task, msecs_to_jiffies(30));
3576 return;
3577 err_mem:
3578 i40evf_free_rss(adapter);
3579 err_register:
3580 i40evf_free_misc_irq(adapter);
3581 err_sw_init:
3582 i40evf_reset_interrupt_capability(adapter);
3583 err_alloc:
3584 kfree(adapter->vf_res);
3585 adapter->vf_res = NULL;
3586 err:
3587 /* Things went into the weeds, so try again later */
3588 if (++adapter->aq_wait_count > I40EVF_AQ_MAX_ERR) {
3589 dev_err(&pdev->dev, "Failed to communicate with PF; waiting before retry\n");
3590 adapter->flags |= I40EVF_FLAG_PF_COMMS_FAILED;
3591 i40evf_shutdown_adminq(hw);
3592 adapter->state = __I40EVF_STARTUP;
3593 schedule_delayed_work(&adapter->init_task, HZ * 5);
3594 return;
3595 }
3596 schedule_delayed_work(&adapter->init_task, HZ);
3597 }
3599 /**
3600 * i40evf_shutdown - Shutdown the device in preparation for a reboot
3601 * @pdev: pci device structure
3602 **/
3603 static void i40evf_shutdown(struct pci_dev *pdev)
3604 {
3605 struct net_device *netdev = pci_get_drvdata(pdev);
3606 struct i40evf_adapter *adapter = netdev_priv(netdev);
3608 netif_device_detach(netdev);
3610 if (netif_running(netdev))
3611 i40evf_close(netdev);
3613 /* Prevent the watchdog from running. */
3614 adapter->state = __I40EVF_REMOVE;
3615 adapter->aq_required = 0;
3617 #ifdef CONFIG_PM
3618 pci_save_state(pdev);
3620 #endif
3621 pci_disable_device(pdev);
3622 }
3624 /**
3625 * i40evf_probe - Device Initialization Routine
3626 * @pdev: PCI device information struct
3627 * @ent: entry in i40evf_pci_tbl
3628 *
3629 * Returns 0 on success, negative on failure
3630 *
3631 * i40evf_probe initializes an adapter identified by a pci_dev structure.
3632 * The OS initialization, configuring of the adapter private structure,
3633 * and a hardware reset occur.
3634 **/
3635 static int i40evf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3636 {
3637 struct net_device *netdev;
3638 struct i40evf_adapter *adapter = NULL;
3639 struct i40e_hw *hw = NULL;
3640 int err;
3642 err = pci_enable_device(pdev);
3643 if (err)
3644 return err;
3646 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3647 if (err) {
3648 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3649 if (err) {
3650 dev_err(&pdev->dev,
3651 "DMA configuration failed: 0x%x\n", err);
3652 goto err_dma;
3653 }
3654 }
3656 err = pci_request_regions(pdev, i40evf_driver_name);
3657 if (err) {
3658 dev_err(&pdev->dev,
3659 "pci_request_regions failed 0x%x\n", err);
3660 goto err_pci_reg;
3661 }
3663 pci_enable_pcie_error_reporting(pdev);
3665 pci_set_master(pdev);
3667 netdev = alloc_etherdev_mq(sizeof(struct i40evf_adapter),
3668 I40EVF_MAX_REQ_QUEUES);
3669 if (!netdev) {
3670 err = -ENOMEM;
3671 goto err_alloc_etherdev;
3672 }
3674 SET_NETDEV_DEV(netdev, &pdev->dev);
3676 pci_set_drvdata(pdev, netdev);
3677 adapter = netdev_priv(netdev);
3679 adapter->netdev = netdev;
3680 adapter->pdev = pdev;
3682 hw = &adapter->hw;
3683 hw->back = adapter;
3685 adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
3686 adapter->state = __I40EVF_STARTUP;
3688 /* Call save state here because it relies on the adapter struct. */
3689 pci_save_state(pdev);
3691 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3692 pci_resource_len(pdev, 0));
3693 if (!hw->hw_addr) {
3694 err = -EIO;
3695 goto err_ioremap;
3696 }
3697 hw->vendor_id = pdev->vendor;
3698 hw->device_id = pdev->device;
3699 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
3700 hw->subsystem_vendor_id = pdev->subsystem_vendor;
3701 hw->subsystem_device_id = pdev->subsystem_device;
3702 hw->bus.device = PCI_SLOT(pdev->devfn);
3703 hw->bus.func = PCI_FUNC(pdev->devfn);
3704 hw->bus.bus_id = pdev->bus->number;
3706 /* set up the locks for the AQ, do this only once in probe
3707 * and destroy them only once in remove
3708 */
3709 mutex_init(&hw->aq.asq_mutex);
3710 mutex_init(&hw->aq.arq_mutex);
3712 spin_lock_init(&adapter->mac_vlan_list_lock);
3713 spin_lock_init(&adapter->cloud_filter_list_lock);
3715 INIT_LIST_HEAD(&adapter->mac_filter_list);
3716 INIT_LIST_HEAD(&adapter->vlan_filter_list);
3717 INIT_LIST_HEAD(&adapter->cloud_filter_list);
3719 INIT_WORK(&adapter->reset_task, i40evf_reset_task);
3720 INIT_WORK(&adapter->adminq_task, i40evf_adminq_task);
3721 INIT_WORK(&adapter->watchdog_task, i40evf_watchdog_task);
3722 INIT_DELAYED_WORK(&adapter->client_task, i40evf_client_task);
3723 INIT_DELAYED_WORK(&adapter->init_task, i40evf_init_task);
3724 schedule_delayed_work(&adapter->init_task,
3725 msecs_to_jiffies(5 * (pdev->devfn & 0x07)));
3727 /* Setup the wait queue for indicating transition to down status */
3728 init_waitqueue_head(&adapter->down_waitqueue);
3730 return 0;
3732 err_ioremap:
3733 free_netdev(netdev);
3734 err_alloc_etherdev:
3735 pci_release_regions(pdev);
3736 err_pci_reg:
3737 err_dma:
3738 pci_disable_device(pdev);
3739 return err;
3740 }
3742 #ifdef CONFIG_PM
3743 /**
3744 * i40evf_suspend - Power management suspend routine
3745 * @pdev: PCI device information struct
3746 * @state: unused
3747 *
3748 * Called when the system (VM) is entering sleep/suspend.
3749 **/
3750 static int i40evf_suspend(struct pci_dev *pdev, pm_message_t state)
3751 {
3752 struct net_device *netdev = pci_get_drvdata(pdev);
3753 struct i40evf_adapter *adapter = netdev_priv(netdev);
3754 int retval = 0;
3756 netif_device_detach(netdev);
3758 while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
3759 &adapter->crit_section))
3760 usleep_range(500, 1000);
3762 if (netif_running(netdev)) {
3763 rtnl_lock();
3764 i40evf_down(adapter);
3765 rtnl_unlock();
3766 }
3767 i40evf_free_misc_irq(adapter);
3768 i40evf_reset_interrupt_capability(adapter);
3770 clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
3772 retval = pci_save_state(pdev);
3773 if (retval)
3774 return retval;
3776 pci_disable_device(pdev);
3778 return 0;
3779 }
3781 /**
3782 * i40evf_resume - Power management resume routine
3783 * @pdev: PCI device information struct
3784 *
3785 * Called when the system (VM) is resumed from sleep/suspend.
3786 **/
3787 static int i40evf_resume(struct pci_dev *pdev)
3788 {
3789 struct i40evf_adapter *adapter = pci_get_drvdata(pdev);
3790 struct net_device *netdev = adapter->netdev;
3791 u32 err;
3793 pci_set_power_state(pdev, PCI_D0);
3794 pci_restore_state(pdev);
3795 /* pci_restore_state clears dev->state_saved so call
3796 * pci_save_state to restore it.
3797 */
3798 pci_save_state(pdev);
3800 err = pci_enable_device_mem(pdev);
3801 if (err) {
3802 dev_err(&pdev->dev, "Cannot enable PCI device from suspend.\n");
3803 return err;
3804 }
3805 pci_set_master(pdev);
3807 rtnl_lock();
3808 err = i40evf_set_interrupt_capability(adapter);
3809 if (err) {
3810 rtnl_unlock();
3811 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n");
3812 return err;
3813 }
3814 err = i40evf_request_misc_irq(adapter);
3815 rtnl_unlock();
3816 if (err) {
3817 dev_err(&pdev->dev, "Cannot get interrupt vector.\n");
3818 return err;
3819 }
3821 schedule_work(&adapter->reset_task);
3823 netif_device_attach(netdev);
3825 return err;
3826 }
3828 #endif /* CONFIG_PM */
3829 /**
3830 * i40evf_remove - Device Removal Routine
3831 * @pdev: PCI device information struct
3832 *
3833 * i40evf_remove is called by the PCI subsystem to alert the driver
3834 * that it should release a PCI device. The could be caused by a
3835 * Hot-Plug event, or because the driver is going to be removed from
3836 * memory.
3837 **/
3838 static void i40evf_remove(struct pci_dev *pdev)
3839 {
3840 struct net_device *netdev = pci_get_drvdata(pdev);
3841 struct i40evf_adapter *adapter = netdev_priv(netdev);
3842 struct i40evf_vlan_filter *vlf, *vlftmp;
3843 struct i40evf_mac_filter *f, *ftmp;
3844 struct i40evf_cloud_filter *cf, *cftmp;
3845 struct i40e_hw *hw = &adapter->hw;
3846 int err;
3847 /* Indicate we are in remove and not to run reset_task */
3848 set_bit(__I40EVF_IN_REMOVE_TASK, &adapter->crit_section);
3849 cancel_delayed_work_sync(&adapter->init_task);
3850 cancel_work_sync(&adapter->reset_task);
3851 cancel_delayed_work_sync(&adapter->client_task);
3852 if (adapter->netdev_registered) {
3853 unregister_netdev(netdev);
3854 adapter->netdev_registered = false;
3855 }
3856 if (CLIENT_ALLOWED(adapter)) {
3857 err = i40evf_lan_del_device(adapter);
3858 if (err)
3859 dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
3860 err);
3861 }
3863 /* Shut down all the garbage mashers on the detention level */
3864 adapter->state = __I40EVF_REMOVE;
3865 adapter->aq_required = 0;
3866 adapter->flags &= ~I40EVF_FLAG_REINIT_ITR_NEEDED;
3867 i40evf_request_reset(adapter);
3868 msleep(50);
3869 /* If the FW isn't responding, kick it once, but only once. */
3870 if (!i40evf_asq_done(hw)) {
3871 i40evf_request_reset(adapter);
3872 msleep(50);
3873 }
3874 i40evf_free_all_tx_resources(adapter);
3875 i40evf_free_all_rx_resources(adapter);
3876 i40evf_misc_irq_disable(adapter);
3877 i40evf_free_misc_irq(adapter);
3878 i40evf_reset_interrupt_capability(adapter);
3879 i40evf_free_q_vectors(adapter);
3881 if (adapter->watchdog_timer.function)
3882 del_timer_sync(&adapter->watchdog_timer);
3884 i40evf_free_rss(adapter);
3886 if (hw->aq.asq.count)
3887 i40evf_shutdown_adminq(hw);
3889 /* destroy the locks only once, here */
3890 mutex_destroy(&hw->aq.arq_mutex);
3891 mutex_destroy(&hw->aq.asq_mutex);
3893 iounmap(hw->hw_addr);
3894 pci_release_regions(pdev);
3895 i40evf_free_all_tx_resources(adapter);
3896 i40evf_free_all_rx_resources(adapter);
3897 i40evf_free_queues(adapter);
3898 kfree(adapter->vf_res);
3899 spin_lock_bh(&adapter->mac_vlan_list_lock);
3900 /* If we got removed before an up/down sequence, we've got a filter
3901 * hanging out there that we need to get rid of.
3902 */
3903 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
3904 list_del(&f->list);
3905 kfree(f);
3906 }
3907 list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
3908 list) {
3909 list_del(&vlf->list);
3910 kfree(vlf);
3911 }
3913 spin_unlock_bh(&adapter->mac_vlan_list_lock);
3915 spin_lock_bh(&adapter->cloud_filter_list_lock);
3916 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
3917 list_del(&cf->list);
3918 kfree(cf);
3919 }
3920 spin_unlock_bh(&adapter->cloud_filter_list_lock);
3922 free_netdev(netdev);
3924 pci_disable_pcie_error_reporting(pdev);
3926 pci_disable_device(pdev);
3927 }
3929 static struct pci_driver i40evf_driver = {
3930 .name = i40evf_driver_name,
3931 .id_table = i40evf_pci_tbl,
3932 .probe = i40evf_probe,
3933 .remove = i40evf_remove,
3934 #ifdef CONFIG_PM
3935 .suspend = i40evf_suspend,
3936 .resume = i40evf_resume,
3937 #endif
3938 .shutdown = i40evf_shutdown,
3939 };
3941 /**
3942 * i40e_init_module - Driver Registration Routine
3943 *
3944 * i40e_init_module is the first routine called when the driver is
3945 * loaded. All it does is register with the PCI subsystem.
3946 **/
3947 static int __init i40evf_init_module(void)
3948 {
3949 int ret;
3951 pr_info("i40evf: %s - version %s\n", i40evf_driver_string,
3952 i40evf_driver_version);
3954 pr_info("%s\n", i40evf_copyright);
3956 i40evf_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1,
3957 i40evf_driver_name);
3958 if (!i40evf_wq) {
3959 pr_err("%s: Failed to create workqueue\n", i40evf_driver_name);
3960 return -ENOMEM;
3961 }
3962 ret = pci_register_driver(&i40evf_driver);
3963 return ret;
3964 }
3966 module_init(i40evf_init_module);
3968 /**
3969 * i40e_exit_module - Driver Exit Cleanup Routine
3970 *
3971 * i40e_exit_module is called just before the driver is removed
3972 * from memory.
3973 **/
3974 static void __exit i40evf_exit_module(void)
3975 {
3976 pci_unregister_driver(&i40evf_driver);
3977 destroy_workqueue(i40evf_wq);
3978 }
3980 module_exit(i40evf_exit_module);
3982 /* i40evf_main.c */