diff options
Diffstat (limited to 'net/core/dev.c')
-rw-r--r-- | net/core/dev.c | 135 |
1 files changed, 87 insertions, 48 deletions
diff --git a/net/core/dev.c b/net/core/dev.c index 6f203c7fb166..24d243084aab 100644 --- a/net/core/dev.c +++ b/net/core/dev.c | |||
@@ -182,7 +182,7 @@ EXPORT_SYMBOL(dev_base_lock); | |||
182 | /* protects napi_hash addition/deletion and napi_gen_id */ | 182 | /* protects napi_hash addition/deletion and napi_gen_id */ |
183 | static DEFINE_SPINLOCK(napi_hash_lock); | 183 | static DEFINE_SPINLOCK(napi_hash_lock); |
184 | 184 | ||
185 | static unsigned int napi_gen_id; | 185 | static unsigned int napi_gen_id = NR_CPUS; |
186 | static DEFINE_HASHTABLE(napi_hash, 8); | 186 | static DEFINE_HASHTABLE(napi_hash, 8); |
187 | 187 | ||
188 | static seqcount_t devnet_rename_seq; | 188 | static seqcount_t devnet_rename_seq; |
@@ -1246,8 +1246,9 @@ int dev_set_alias(struct net_device *dev, const char *alias, size_t len) | |||
1246 | if (!new_ifalias) | 1246 | if (!new_ifalias) |
1247 | return -ENOMEM; | 1247 | return -ENOMEM; |
1248 | dev->ifalias = new_ifalias; | 1248 | dev->ifalias = new_ifalias; |
1249 | memcpy(dev->ifalias, alias, len); | ||
1250 | dev->ifalias[len] = 0; | ||
1249 | 1251 | ||
1250 | strlcpy(dev->ifalias, alias, len+1); | ||
1251 | return len; | 1252 | return len; |
1252 | } | 1253 | } |
1253 | 1254 | ||
@@ -1676,37 +1677,59 @@ EXPORT_SYMBOL_GPL(net_dec_ingress_queue); | |||
1676 | 1677 | ||
1677 | static struct static_key netstamp_needed __read_mostly; | 1678 | static struct static_key netstamp_needed __read_mostly; |
1678 | #ifdef HAVE_JUMP_LABEL | 1679 | #ifdef HAVE_JUMP_LABEL |
1679 | /* We are not allowed to call static_key_slow_dec() from irq context | ||
1680 | * If net_disable_timestamp() is called from irq context, defer the | ||
1681 | * static_key_slow_dec() calls. | ||
1682 | */ | ||
1683 | static atomic_t netstamp_needed_deferred; | 1680 | static atomic_t netstamp_needed_deferred; |
1681 | static atomic_t netstamp_wanted; | ||
1682 | static void netstamp_clear(struct work_struct *work) | ||
1683 | { | ||
1684 | int deferred = atomic_xchg(&netstamp_needed_deferred, 0); | ||
1685 | int wanted; | ||
1686 | |||
1687 | wanted = atomic_add_return(deferred, &netstamp_wanted); | ||
1688 | if (wanted > 0) | ||
1689 | static_key_enable(&netstamp_needed); | ||
1690 | else | ||
1691 | static_key_disable(&netstamp_needed); | ||
1692 | } | ||
1693 | static DECLARE_WORK(netstamp_work, netstamp_clear); | ||
1684 | #endif | 1694 | #endif |
1685 | 1695 | ||
1686 | void net_enable_timestamp(void) | 1696 | void net_enable_timestamp(void) |
1687 | { | 1697 | { |
1688 | #ifdef HAVE_JUMP_LABEL | 1698 | #ifdef HAVE_JUMP_LABEL |
1689 | int deferred = atomic_xchg(&netstamp_needed_deferred, 0); | 1699 | int wanted; |
1690 | 1700 | ||
1691 | if (deferred) { | 1701 | while (1) { |
1692 | while (--deferred) | 1702 | wanted = atomic_read(&netstamp_wanted); |
1693 | static_key_slow_dec(&netstamp_needed); | 1703 | if (wanted <= 0) |
1694 | return; | 1704 | break; |
1705 | if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted) | ||
1706 | return; | ||
1695 | } | 1707 | } |
1696 | #endif | 1708 | atomic_inc(&netstamp_needed_deferred); |
1709 | schedule_work(&netstamp_work); | ||
1710 | #else | ||
1697 | static_key_slow_inc(&netstamp_needed); | 1711 | static_key_slow_inc(&netstamp_needed); |
1712 | #endif | ||
1698 | } | 1713 | } |
1699 | EXPORT_SYMBOL(net_enable_timestamp); | 1714 | EXPORT_SYMBOL(net_enable_timestamp); |
1700 | 1715 | ||
1701 | void net_disable_timestamp(void) | 1716 | void net_disable_timestamp(void) |
1702 | { | 1717 | { |
1703 | #ifdef HAVE_JUMP_LABEL | 1718 | #ifdef HAVE_JUMP_LABEL |
1704 | if (in_interrupt()) { | 1719 | int wanted; |
1705 | atomic_inc(&netstamp_needed_deferred); | 1720 | |
1706 | return; | 1721 | while (1) { |
1722 | wanted = atomic_read(&netstamp_wanted); | ||
1723 | if (wanted <= 1) | ||
1724 | break; | ||
1725 | if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted) | ||
1726 | return; | ||
1707 | } | 1727 | } |
1708 | #endif | 1728 | atomic_dec(&netstamp_needed_deferred); |
1729 | schedule_work(&netstamp_work); | ||
1730 | #else | ||
1709 | static_key_slow_dec(&netstamp_needed); | 1731 | static_key_slow_dec(&netstamp_needed); |
1732 | #endif | ||
1710 | } | 1733 | } |
1711 | EXPORT_SYMBOL(net_disable_timestamp); | 1734 | EXPORT_SYMBOL(net_disable_timestamp); |
1712 | 1735 | ||
@@ -2527,9 +2550,10 @@ EXPORT_SYMBOL(skb_mac_gso_segment); | |||
2527 | static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) | 2550 | static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) |
2528 | { | 2551 | { |
2529 | if (tx_path) | 2552 | if (tx_path) |
2530 | return skb->ip_summed != CHECKSUM_PARTIAL; | 2553 | return skb->ip_summed != CHECKSUM_PARTIAL && |
2531 | else | 2554 | skb->ip_summed != CHECKSUM_UNNECESSARY; |
2532 | return skb->ip_summed == CHECKSUM_NONE; | 2555 | |
2556 | return skb->ip_summed == CHECKSUM_NONE; | ||
2533 | } | 2557 | } |
2534 | 2558 | ||
2535 | /** | 2559 | /** |
@@ -2548,11 +2572,12 @@ static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) | |||
2548 | struct sk_buff *__skb_gso_segment(struct sk_buff *skb, | 2572 | struct sk_buff *__skb_gso_segment(struct sk_buff *skb, |
2549 | netdev_features_t features, bool tx_path) | 2573 | netdev_features_t features, bool tx_path) |
2550 | { | 2574 | { |
2575 | struct sk_buff *segs; | ||
2576 | |||
2551 | if (unlikely(skb_needs_check(skb, tx_path))) { | 2577 | if (unlikely(skb_needs_check(skb, tx_path))) { |
2552 | int err; | 2578 | int err; |
2553 | 2579 | ||
2554 | skb_warn_bad_offload(skb); | 2580 | /* We're going to init ->check field in TCP or UDP header */ |
2555 | |||
2556 | err = skb_cow_head(skb, 0); | 2581 | err = skb_cow_head(skb, 0); |
2557 | if (err < 0) | 2582 | if (err < 0) |
2558 | return ERR_PTR(err); | 2583 | return ERR_PTR(err); |
@@ -2567,7 +2592,12 @@ struct sk_buff *__skb_gso_segment(struct sk_buff *skb, | |||
2567 | skb_reset_mac_header(skb); | 2592 | skb_reset_mac_header(skb); |
2568 | skb_reset_mac_len(skb); | 2593 | skb_reset_mac_len(skb); |
2569 | 2594 | ||
2570 | return skb_mac_gso_segment(skb, features); | 2595 | segs = skb_mac_gso_segment(skb, features); |
2596 | |||
2597 | if (unlikely(skb_needs_check(skb, tx_path))) | ||
2598 | skb_warn_bad_offload(skb); | ||
2599 | |||
2600 | return segs; | ||
2571 | } | 2601 | } |
2572 | EXPORT_SYMBOL(__skb_gso_segment); | 2602 | EXPORT_SYMBOL(__skb_gso_segment); |
2573 | 2603 | ||
@@ -2650,9 +2680,9 @@ static netdev_features_t harmonize_features(struct sk_buff *skb, | |||
2650 | if (skb->ip_summed != CHECKSUM_NONE && | 2680 | if (skb->ip_summed != CHECKSUM_NONE && |
2651 | !can_checksum_protocol(features, type)) { | 2681 | !can_checksum_protocol(features, type)) { |
2652 | features &= ~NETIF_F_ALL_CSUM; | 2682 | features &= ~NETIF_F_ALL_CSUM; |
2653 | } else if (illegal_highdma(skb->dev, skb)) { | ||
2654 | features &= ~NETIF_F_SG; | ||
2655 | } | 2683 | } |
2684 | if (illegal_highdma(skb->dev, skb)) | ||
2685 | features &= ~NETIF_F_SG; | ||
2656 | 2686 | ||
2657 | return features; | 2687 | return features; |
2658 | } | 2688 | } |
@@ -3027,7 +3057,9 @@ struct netdev_queue *netdev_pick_tx(struct net_device *dev, | |||
3027 | int queue_index = 0; | 3057 | int queue_index = 0; |
3028 | 3058 | ||
3029 | #ifdef CONFIG_XPS | 3059 | #ifdef CONFIG_XPS |
3030 | if (skb->sender_cpu == 0) | 3060 | u32 sender_cpu = skb->sender_cpu - 1; |
3061 | |||
3062 | if (sender_cpu >= (u32)NR_CPUS) | ||
3031 | skb->sender_cpu = raw_smp_processor_id() + 1; | 3063 | skb->sender_cpu = raw_smp_processor_id() + 1; |
3032 | #endif | 3064 | #endif |
3033 | 3065 | ||
@@ -4350,6 +4382,12 @@ struct packet_offload *gro_find_complete_by_type(__be16 type) | |||
4350 | } | 4382 | } |
4351 | EXPORT_SYMBOL(gro_find_complete_by_type); | 4383 | EXPORT_SYMBOL(gro_find_complete_by_type); |
4352 | 4384 | ||
4385 | static void napi_skb_free_stolen_head(struct sk_buff *skb) | ||
4386 | { | ||
4387 | skb_dst_drop(skb); | ||
4388 | kmem_cache_free(skbuff_head_cache, skb); | ||
4389 | } | ||
4390 | |||
4353 | static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) | 4391 | static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) |
4354 | { | 4392 | { |
4355 | switch (ret) { | 4393 | switch (ret) { |
@@ -4363,12 +4401,10 @@ static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) | |||
4363 | break; | 4401 | break; |
4364 | 4402 | ||
4365 | case GRO_MERGED_FREE: | 4403 | case GRO_MERGED_FREE: |
4366 | if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) { | 4404 | if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) |
4367 | skb_dst_drop(skb); | 4405 | napi_skb_free_stolen_head(skb); |
4368 | kmem_cache_free(skbuff_head_cache, skb); | 4406 | else |
4369 | } else { | ||
4370 | __kfree_skb(skb); | 4407 | __kfree_skb(skb); |
4371 | } | ||
4372 | break; | 4408 | break; |
4373 | 4409 | ||
4374 | case GRO_HELD: | 4410 | case GRO_HELD: |
@@ -4434,10 +4470,16 @@ static gro_result_t napi_frags_finish(struct napi_struct *napi, | |||
4434 | break; | 4470 | break; |
4435 | 4471 | ||
4436 | case GRO_DROP: | 4472 | case GRO_DROP: |
4437 | case GRO_MERGED_FREE: | ||
4438 | napi_reuse_skb(napi, skb); | 4473 | napi_reuse_skb(napi, skb); |
4439 | break; | 4474 | break; |
4440 | 4475 | ||
4476 | case GRO_MERGED_FREE: | ||
4477 | if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) | ||
4478 | napi_skb_free_stolen_head(skb); | ||
4479 | else | ||
4480 | napi_reuse_skb(napi, skb); | ||
4481 | break; | ||
4482 | |||
4441 | case GRO_MERGED: | 4483 | case GRO_MERGED: |
4442 | break; | 4484 | break; |
4443 | } | 4485 | } |
@@ -4704,25 +4746,22 @@ EXPORT_SYMBOL_GPL(napi_by_id); | |||
4704 | 4746 | ||
4705 | void napi_hash_add(struct napi_struct *napi) | 4747 | void napi_hash_add(struct napi_struct *napi) |
4706 | { | 4748 | { |
4707 | if (!test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) { | 4749 | if (test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) |
4750 | return; | ||
4708 | 4751 | ||
4709 | spin_lock(&napi_hash_lock); | 4752 | spin_lock(&napi_hash_lock); |
4710 | 4753 | ||
4711 | /* 0 is not a valid id, we also skip an id that is taken | 4754 | /* 0..NR_CPUS+1 range is reserved for sender_cpu use */ |
4712 | * we expect both events to be extremely rare | 4755 | do { |
4713 | */ | 4756 | if (unlikely(++napi_gen_id < NR_CPUS + 1)) |
4714 | napi->napi_id = 0; | 4757 | napi_gen_id = NR_CPUS + 1; |
4715 | while (!napi->napi_id) { | 4758 | } while (napi_by_id(napi_gen_id)); |
4716 | napi->napi_id = ++napi_gen_id; | 4759 | napi->napi_id = napi_gen_id; |
4717 | if (napi_by_id(napi->napi_id)) | ||
4718 | napi->napi_id = 0; | ||
4719 | } | ||
4720 | 4760 | ||
4721 | hlist_add_head_rcu(&napi->napi_hash_node, | 4761 | hlist_add_head_rcu(&napi->napi_hash_node, |
4722 | &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); | 4762 | &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); |
4723 | 4763 | ||
4724 | spin_unlock(&napi_hash_lock); | 4764 | spin_unlock(&napi_hash_lock); |
4725 | } | ||
4726 | } | 4765 | } |
4727 | EXPORT_SYMBOL_GPL(napi_hash_add); | 4766 | EXPORT_SYMBOL_GPL(napi_hash_add); |
4728 | 4767 | ||
@@ -7030,8 +7069,8 @@ struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, | |||
7030 | } else { | 7069 | } else { |
7031 | netdev_stats_to_stats64(storage, &dev->stats); | 7070 | netdev_stats_to_stats64(storage, &dev->stats); |
7032 | } | 7071 | } |
7033 | storage->rx_dropped += atomic_long_read(&dev->rx_dropped); | 7072 | storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped); |
7034 | storage->tx_dropped += atomic_long_read(&dev->tx_dropped); | 7073 | storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped); |
7035 | return storage; | 7074 | return storage; |
7036 | } | 7075 | } |
7037 | EXPORT_SYMBOL(dev_get_stats); | 7076 | EXPORT_SYMBOL(dev_get_stats); |