aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat (limited to 'net/core/dev.c')
-rw-r--r--net/core/dev.c135
1 files changed, 87 insertions, 48 deletions
diff --git a/net/core/dev.c b/net/core/dev.c
index 6f203c7fb166..24d243084aab 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -182,7 +182,7 @@ EXPORT_SYMBOL(dev_base_lock);
182/* protects napi_hash addition/deletion and napi_gen_id */ 182/* protects napi_hash addition/deletion and napi_gen_id */
183static DEFINE_SPINLOCK(napi_hash_lock); 183static DEFINE_SPINLOCK(napi_hash_lock);
184 184
185static unsigned int napi_gen_id; 185static unsigned int napi_gen_id = NR_CPUS;
186static DEFINE_HASHTABLE(napi_hash, 8); 186static DEFINE_HASHTABLE(napi_hash, 8);
187 187
188static seqcount_t devnet_rename_seq; 188static seqcount_t devnet_rename_seq;
@@ -1246,8 +1246,9 @@ int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1246 if (!new_ifalias) 1246 if (!new_ifalias)
1247 return -ENOMEM; 1247 return -ENOMEM;
1248 dev->ifalias = new_ifalias; 1248 dev->ifalias = new_ifalias;
1249 memcpy(dev->ifalias, alias, len);
1250 dev->ifalias[len] = 0;
1249 1251
1250 strlcpy(dev->ifalias, alias, len+1);
1251 return len; 1252 return len;
1252} 1253}
1253 1254
@@ -1676,37 +1677,59 @@ EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1676 1677
1677static struct static_key netstamp_needed __read_mostly; 1678static struct static_key netstamp_needed __read_mostly;
1678#ifdef HAVE_JUMP_LABEL 1679#ifdef HAVE_JUMP_LABEL
1679/* We are not allowed to call static_key_slow_dec() from irq context
1680 * If net_disable_timestamp() is called from irq context, defer the
1681 * static_key_slow_dec() calls.
1682 */
1683static atomic_t netstamp_needed_deferred; 1680static atomic_t netstamp_needed_deferred;
1681static atomic_t netstamp_wanted;
1682static void netstamp_clear(struct work_struct *work)
1683{
1684 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1685 int wanted;
1686
1687 wanted = atomic_add_return(deferred, &netstamp_wanted);
1688 if (wanted > 0)
1689 static_key_enable(&netstamp_needed);
1690 else
1691 static_key_disable(&netstamp_needed);
1692}
1693static DECLARE_WORK(netstamp_work, netstamp_clear);
1684#endif 1694#endif
1685 1695
1686void net_enable_timestamp(void) 1696void net_enable_timestamp(void)
1687{ 1697{
1688#ifdef HAVE_JUMP_LABEL 1698#ifdef HAVE_JUMP_LABEL
1689 int deferred = atomic_xchg(&netstamp_needed_deferred, 0); 1699 int wanted;
1690 1700
1691 if (deferred) { 1701 while (1) {
1692 while (--deferred) 1702 wanted = atomic_read(&netstamp_wanted);
1693 static_key_slow_dec(&netstamp_needed); 1703 if (wanted <= 0)
1694 return; 1704 break;
1705 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
1706 return;
1695 } 1707 }
1696#endif 1708 atomic_inc(&netstamp_needed_deferred);
1709 schedule_work(&netstamp_work);
1710#else
1697 static_key_slow_inc(&netstamp_needed); 1711 static_key_slow_inc(&netstamp_needed);
1712#endif
1698} 1713}
1699EXPORT_SYMBOL(net_enable_timestamp); 1714EXPORT_SYMBOL(net_enable_timestamp);
1700 1715
1701void net_disable_timestamp(void) 1716void net_disable_timestamp(void)
1702{ 1717{
1703#ifdef HAVE_JUMP_LABEL 1718#ifdef HAVE_JUMP_LABEL
1704 if (in_interrupt()) { 1719 int wanted;
1705 atomic_inc(&netstamp_needed_deferred); 1720
1706 return; 1721 while (1) {
1722 wanted = atomic_read(&netstamp_wanted);
1723 if (wanted <= 1)
1724 break;
1725 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
1726 return;
1707 } 1727 }
1708#endif 1728 atomic_dec(&netstamp_needed_deferred);
1729 schedule_work(&netstamp_work);
1730#else
1709 static_key_slow_dec(&netstamp_needed); 1731 static_key_slow_dec(&netstamp_needed);
1732#endif
1710} 1733}
1711EXPORT_SYMBOL(net_disable_timestamp); 1734EXPORT_SYMBOL(net_disable_timestamp);
1712 1735
@@ -2527,9 +2550,10 @@ EXPORT_SYMBOL(skb_mac_gso_segment);
2527static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) 2550static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2528{ 2551{
2529 if (tx_path) 2552 if (tx_path)
2530 return skb->ip_summed != CHECKSUM_PARTIAL; 2553 return skb->ip_summed != CHECKSUM_PARTIAL &&
2531 else 2554 skb->ip_summed != CHECKSUM_UNNECESSARY;
2532 return skb->ip_summed == CHECKSUM_NONE; 2555
2556 return skb->ip_summed == CHECKSUM_NONE;
2533} 2557}
2534 2558
2535/** 2559/**
@@ -2548,11 +2572,12 @@ static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2548struct sk_buff *__skb_gso_segment(struct sk_buff *skb, 2572struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2549 netdev_features_t features, bool tx_path) 2573 netdev_features_t features, bool tx_path)
2550{ 2574{
2575 struct sk_buff *segs;
2576
2551 if (unlikely(skb_needs_check(skb, tx_path))) { 2577 if (unlikely(skb_needs_check(skb, tx_path))) {
2552 int err; 2578 int err;
2553 2579
2554 skb_warn_bad_offload(skb); 2580 /* We're going to init ->check field in TCP or UDP header */
2555
2556 err = skb_cow_head(skb, 0); 2581 err = skb_cow_head(skb, 0);
2557 if (err < 0) 2582 if (err < 0)
2558 return ERR_PTR(err); 2583 return ERR_PTR(err);
@@ -2567,7 +2592,12 @@ struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2567 skb_reset_mac_header(skb); 2592 skb_reset_mac_header(skb);
2568 skb_reset_mac_len(skb); 2593 skb_reset_mac_len(skb);
2569 2594
2570 return skb_mac_gso_segment(skb, features); 2595 segs = skb_mac_gso_segment(skb, features);
2596
2597 if (unlikely(skb_needs_check(skb, tx_path)))
2598 skb_warn_bad_offload(skb);
2599
2600 return segs;
2571} 2601}
2572EXPORT_SYMBOL(__skb_gso_segment); 2602EXPORT_SYMBOL(__skb_gso_segment);
2573 2603
@@ -2650,9 +2680,9 @@ static netdev_features_t harmonize_features(struct sk_buff *skb,
2650 if (skb->ip_summed != CHECKSUM_NONE && 2680 if (skb->ip_summed != CHECKSUM_NONE &&
2651 !can_checksum_protocol(features, type)) { 2681 !can_checksum_protocol(features, type)) {
2652 features &= ~NETIF_F_ALL_CSUM; 2682 features &= ~NETIF_F_ALL_CSUM;
2653 } else if (illegal_highdma(skb->dev, skb)) {
2654 features &= ~NETIF_F_SG;
2655 } 2683 }
2684 if (illegal_highdma(skb->dev, skb))
2685 features &= ~NETIF_F_SG;
2656 2686
2657 return features; 2687 return features;
2658} 2688}
@@ -3027,7 +3057,9 @@ struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3027 int queue_index = 0; 3057 int queue_index = 0;
3028 3058
3029#ifdef CONFIG_XPS 3059#ifdef CONFIG_XPS
3030 if (skb->sender_cpu == 0) 3060 u32 sender_cpu = skb->sender_cpu - 1;
3061
3062 if (sender_cpu >= (u32)NR_CPUS)
3031 skb->sender_cpu = raw_smp_processor_id() + 1; 3063 skb->sender_cpu = raw_smp_processor_id() + 1;
3032#endif 3064#endif
3033 3065
@@ -4350,6 +4382,12 @@ struct packet_offload *gro_find_complete_by_type(__be16 type)
4350} 4382}
4351EXPORT_SYMBOL(gro_find_complete_by_type); 4383EXPORT_SYMBOL(gro_find_complete_by_type);
4352 4384
4385static void napi_skb_free_stolen_head(struct sk_buff *skb)
4386{
4387 skb_dst_drop(skb);
4388 kmem_cache_free(skbuff_head_cache, skb);
4389}
4390
4353static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) 4391static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
4354{ 4392{
4355 switch (ret) { 4393 switch (ret) {
@@ -4363,12 +4401,10 @@ static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
4363 break; 4401 break;
4364 4402
4365 case GRO_MERGED_FREE: 4403 case GRO_MERGED_FREE:
4366 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) { 4404 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
4367 skb_dst_drop(skb); 4405 napi_skb_free_stolen_head(skb);
4368 kmem_cache_free(skbuff_head_cache, skb); 4406 else
4369 } else {
4370 __kfree_skb(skb); 4407 __kfree_skb(skb);
4371 }
4372 break; 4408 break;
4373 4409
4374 case GRO_HELD: 4410 case GRO_HELD:
@@ -4434,10 +4470,16 @@ static gro_result_t napi_frags_finish(struct napi_struct *napi,
4434 break; 4470 break;
4435 4471
4436 case GRO_DROP: 4472 case GRO_DROP:
4437 case GRO_MERGED_FREE:
4438 napi_reuse_skb(napi, skb); 4473 napi_reuse_skb(napi, skb);
4439 break; 4474 break;
4440 4475
4476 case GRO_MERGED_FREE:
4477 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
4478 napi_skb_free_stolen_head(skb);
4479 else
4480 napi_reuse_skb(napi, skb);
4481 break;
4482
4441 case GRO_MERGED: 4483 case GRO_MERGED:
4442 break; 4484 break;
4443 } 4485 }
@@ -4704,25 +4746,22 @@ EXPORT_SYMBOL_GPL(napi_by_id);
4704 4746
4705void napi_hash_add(struct napi_struct *napi) 4747void napi_hash_add(struct napi_struct *napi)
4706{ 4748{
4707 if (!test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) { 4749 if (test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
4750 return;
4708 4751
4709 spin_lock(&napi_hash_lock); 4752 spin_lock(&napi_hash_lock);
4710 4753
4711 /* 0 is not a valid id, we also skip an id that is taken 4754 /* 0..NR_CPUS+1 range is reserved for sender_cpu use */
4712 * we expect both events to be extremely rare 4755 do {
4713 */ 4756 if (unlikely(++napi_gen_id < NR_CPUS + 1))
4714 napi->napi_id = 0; 4757 napi_gen_id = NR_CPUS + 1;
4715 while (!napi->napi_id) { 4758 } while (napi_by_id(napi_gen_id));
4716 napi->napi_id = ++napi_gen_id; 4759 napi->napi_id = napi_gen_id;
4717 if (napi_by_id(napi->napi_id))
4718 napi->napi_id = 0;
4719 }
4720 4760
4721 hlist_add_head_rcu(&napi->napi_hash_node, 4761 hlist_add_head_rcu(&napi->napi_hash_node,
4722 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); 4762 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
4723 4763
4724 spin_unlock(&napi_hash_lock); 4764 spin_unlock(&napi_hash_lock);
4725 }
4726} 4765}
4727EXPORT_SYMBOL_GPL(napi_hash_add); 4766EXPORT_SYMBOL_GPL(napi_hash_add);
4728 4767
@@ -7030,8 +7069,8 @@ struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
7030 } else { 7069 } else {
7031 netdev_stats_to_stats64(storage, &dev->stats); 7070 netdev_stats_to_stats64(storage, &dev->stats);
7032 } 7071 }
7033 storage->rx_dropped += atomic_long_read(&dev->rx_dropped); 7072 storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped);
7034 storage->tx_dropped += atomic_long_read(&dev->tx_dropped); 7073 storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped);
7035 return storage; 7074 return storage;
7036} 7075}
7037EXPORT_SYMBOL(dev_get_stats); 7076EXPORT_SYMBOL(dev_get_stats);