diff options
author | Vlastimil Babka | 2017-05-08 17:59:50 -0500 |
---|---|---|
committer | Linus Torvalds | 2017-05-08 19:15:15 -0500 |
commit | 499118e966f1d2150bd66647c8932343c4e9a0b8 (patch) | |
tree | cc3560705304d978526ecffa055066b2c8b09fa6 /mm | |
parent | 62be1511b1db8066220b18b7d4da2e6b9fdc69fb (diff) | |
download | kernel-499118e966f1d2150bd66647c8932343c4e9a0b8.tar.gz kernel-499118e966f1d2150bd66647c8932343c4e9a0b8.tar.xz kernel-499118e966f1d2150bd66647c8932343c4e9a0b8.zip |
mm: introduce memalloc_noreclaim_{save,restore}
The previous patch ("mm: prevent potential recursive reclaim due to
clearing PF_MEMALLOC") has shown that simply setting and clearing
PF_MEMALLOC in current->flags can result in wrongly clearing a
pre-existing PF_MEMALLOC flag and potentially lead to recursive reclaim.
Let's introduce helpers that support proper nesting by saving the
previous stat of the flag, similar to the existing memalloc_noio_* and
memalloc_nofs_* helpers. Convert existing setting/clearing of
PF_MEMALLOC within mm to the new helpers.
There are no known issues with the converted code, but the change makes
it more robust.
Link: http://lkml.kernel.org/r/20170405074700.29871-3-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Suggested-by: Michal Hocko <mhocko@suse.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Boris Brezillon <boris.brezillon@free-electrons.com>
Cc: Chris Leech <cleech@redhat.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Josef Bacik <jbacik@fb.com>
Cc: Lee Duncan <lduncan@suse.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Richard Weinberger <richard@nod.at>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'mm')
-rw-r--r-- | mm/page_alloc.c | 11 | ||||
-rw-r--r-- | mm/vmscan.c | 17 |
2 files changed, 17 insertions, 11 deletions
diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 1daf509722c7..f9e450c6b6e4 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c | |||
@@ -3283,15 +3283,15 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, | |||
3283 | enum compact_priority prio, enum compact_result *compact_result) | 3283 | enum compact_priority prio, enum compact_result *compact_result) |
3284 | { | 3284 | { |
3285 | struct page *page; | 3285 | struct page *page; |
3286 | unsigned int noreclaim_flag = current->flags & PF_MEMALLOC; | 3286 | unsigned int noreclaim_flag; |
3287 | 3287 | ||
3288 | if (!order) | 3288 | if (!order) |
3289 | return NULL; | 3289 | return NULL; |
3290 | 3290 | ||
3291 | current->flags |= PF_MEMALLOC; | 3291 | noreclaim_flag = memalloc_noreclaim_save(); |
3292 | *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, | 3292 | *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, |
3293 | prio); | 3293 | prio); |
3294 | current->flags = (current->flags & ~PF_MEMALLOC) | noreclaim_flag; | 3294 | memalloc_noreclaim_restore(noreclaim_flag); |
3295 | 3295 | ||
3296 | if (*compact_result <= COMPACT_INACTIVE) | 3296 | if (*compact_result <= COMPACT_INACTIVE) |
3297 | return NULL; | 3297 | return NULL; |
@@ -3438,12 +3438,13 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, | |||
3438 | { | 3438 | { |
3439 | struct reclaim_state reclaim_state; | 3439 | struct reclaim_state reclaim_state; |
3440 | int progress; | 3440 | int progress; |
3441 | unsigned int noreclaim_flag; | ||
3441 | 3442 | ||
3442 | cond_resched(); | 3443 | cond_resched(); |
3443 | 3444 | ||
3444 | /* We now go into synchronous reclaim */ | 3445 | /* We now go into synchronous reclaim */ |
3445 | cpuset_memory_pressure_bump(); | 3446 | cpuset_memory_pressure_bump(); |
3446 | current->flags |= PF_MEMALLOC; | 3447 | noreclaim_flag = memalloc_noreclaim_save(); |
3447 | lockdep_set_current_reclaim_state(gfp_mask); | 3448 | lockdep_set_current_reclaim_state(gfp_mask); |
3448 | reclaim_state.reclaimed_slab = 0; | 3449 | reclaim_state.reclaimed_slab = 0; |
3449 | current->reclaim_state = &reclaim_state; | 3450 | current->reclaim_state = &reclaim_state; |
@@ -3453,7 +3454,7 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, | |||
3453 | 3454 | ||
3454 | current->reclaim_state = NULL; | 3455 | current->reclaim_state = NULL; |
3455 | lockdep_clear_current_reclaim_state(); | 3456 | lockdep_clear_current_reclaim_state(); |
3456 | current->flags &= ~PF_MEMALLOC; | 3457 | memalloc_noreclaim_restore(noreclaim_flag); |
3457 | 3458 | ||
3458 | cond_resched(); | 3459 | cond_resched(); |
3459 | 3460 | ||
diff --git a/mm/vmscan.c b/mm/vmscan.c index 4e7ed65842af..2f45c0520f43 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c | |||
@@ -3036,6 +3036,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, | |||
3036 | struct zonelist *zonelist; | 3036 | struct zonelist *zonelist; |
3037 | unsigned long nr_reclaimed; | 3037 | unsigned long nr_reclaimed; |
3038 | int nid; | 3038 | int nid; |
3039 | unsigned int noreclaim_flag; | ||
3039 | struct scan_control sc = { | 3040 | struct scan_control sc = { |
3040 | .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), | 3041 | .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), |
3041 | .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) | | 3042 | .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) | |
@@ -3062,9 +3063,9 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, | |||
3062 | sc.gfp_mask, | 3063 | sc.gfp_mask, |
3063 | sc.reclaim_idx); | 3064 | sc.reclaim_idx); |
3064 | 3065 | ||
3065 | current->flags |= PF_MEMALLOC; | 3066 | noreclaim_flag = memalloc_noreclaim_save(); |
3066 | nr_reclaimed = do_try_to_free_pages(zonelist, &sc); | 3067 | nr_reclaimed = do_try_to_free_pages(zonelist, &sc); |
3067 | current->flags &= ~PF_MEMALLOC; | 3068 | memalloc_noreclaim_restore(noreclaim_flag); |
3068 | 3069 | ||
3069 | trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed); | 3070 | trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed); |
3070 | 3071 | ||
@@ -3589,8 +3590,9 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim) | |||
3589 | struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); | 3590 | struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); |
3590 | struct task_struct *p = current; | 3591 | struct task_struct *p = current; |
3591 | unsigned long nr_reclaimed; | 3592 | unsigned long nr_reclaimed; |
3593 | unsigned int noreclaim_flag; | ||
3592 | 3594 | ||
3593 | p->flags |= PF_MEMALLOC; | 3595 | noreclaim_flag = memalloc_noreclaim_save(); |
3594 | lockdep_set_current_reclaim_state(sc.gfp_mask); | 3596 | lockdep_set_current_reclaim_state(sc.gfp_mask); |
3595 | reclaim_state.reclaimed_slab = 0; | 3597 | reclaim_state.reclaimed_slab = 0; |
3596 | p->reclaim_state = &reclaim_state; | 3598 | p->reclaim_state = &reclaim_state; |
@@ -3599,7 +3601,7 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim) | |||
3599 | 3601 | ||
3600 | p->reclaim_state = NULL; | 3602 | p->reclaim_state = NULL; |
3601 | lockdep_clear_current_reclaim_state(); | 3603 | lockdep_clear_current_reclaim_state(); |
3602 | p->flags &= ~PF_MEMALLOC; | 3604 | memalloc_noreclaim_restore(noreclaim_flag); |
3603 | 3605 | ||
3604 | return nr_reclaimed; | 3606 | return nr_reclaimed; |
3605 | } | 3607 | } |
@@ -3764,6 +3766,7 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in | |||
3764 | struct task_struct *p = current; | 3766 | struct task_struct *p = current; |
3765 | struct reclaim_state reclaim_state; | 3767 | struct reclaim_state reclaim_state; |
3766 | int classzone_idx = gfp_zone(gfp_mask); | 3768 | int classzone_idx = gfp_zone(gfp_mask); |
3769 | unsigned int noreclaim_flag; | ||
3767 | struct scan_control sc = { | 3770 | struct scan_control sc = { |
3768 | .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), | 3771 | .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), |
3769 | .gfp_mask = (gfp_mask = current_gfp_context(gfp_mask)), | 3772 | .gfp_mask = (gfp_mask = current_gfp_context(gfp_mask)), |
@@ -3781,7 +3784,8 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in | |||
3781 | * and we also need to be able to write out pages for RECLAIM_WRITE | 3784 | * and we also need to be able to write out pages for RECLAIM_WRITE |
3782 | * and RECLAIM_UNMAP. | 3785 | * and RECLAIM_UNMAP. |
3783 | */ | 3786 | */ |
3784 | p->flags |= PF_MEMALLOC | PF_SWAPWRITE; | 3787 | noreclaim_flag = memalloc_noreclaim_save(); |
3788 | p->flags |= PF_SWAPWRITE; | ||
3785 | lockdep_set_current_reclaim_state(gfp_mask); | 3789 | lockdep_set_current_reclaim_state(gfp_mask); |
3786 | reclaim_state.reclaimed_slab = 0; | 3790 | reclaim_state.reclaimed_slab = 0; |
3787 | p->reclaim_state = &reclaim_state; | 3791 | p->reclaim_state = &reclaim_state; |
@@ -3797,7 +3801,8 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in | |||
3797 | } | 3801 | } |
3798 | 3802 | ||
3799 | p->reclaim_state = NULL; | 3803 | p->reclaim_state = NULL; |
3800 | current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE); | 3804 | current->flags &= ~PF_SWAPWRITE; |
3805 | memalloc_noreclaim_restore(noreclaim_flag); | ||
3801 | lockdep_clear_current_reclaim_state(); | 3806 | lockdep_clear_current_reclaim_state(); |
3802 | return sc.nr_reclaimed >= nr_pages; | 3807 | return sc.nr_reclaimed >= nr_pages; |
3803 | } | 3808 | } |