aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
authorVlastimil Babka2017-05-08 17:59:50 -0500
committerLinus Torvalds2017-05-08 19:15:15 -0500
commit499118e966f1d2150bd66647c8932343c4e9a0b8 (patch)
treecc3560705304d978526ecffa055066b2c8b09fa6 /mm
parent62be1511b1db8066220b18b7d4da2e6b9fdc69fb (diff)
downloadkernel-499118e966f1d2150bd66647c8932343c4e9a0b8.tar.gz
kernel-499118e966f1d2150bd66647c8932343c4e9a0b8.tar.xz
kernel-499118e966f1d2150bd66647c8932343c4e9a0b8.zip
mm: introduce memalloc_noreclaim_{save,restore}
The previous patch ("mm: prevent potential recursive reclaim due to clearing PF_MEMALLOC") has shown that simply setting and clearing PF_MEMALLOC in current->flags can result in wrongly clearing a pre-existing PF_MEMALLOC flag and potentially lead to recursive reclaim. Let's introduce helpers that support proper nesting by saving the previous stat of the flag, similar to the existing memalloc_noio_* and memalloc_nofs_* helpers. Convert existing setting/clearing of PF_MEMALLOC within mm to the new helpers. There are no known issues with the converted code, but the change makes it more robust. Link: http://lkml.kernel.org/r/20170405074700.29871-3-vbabka@suse.cz Signed-off-by: Vlastimil Babka <vbabka@suse.cz> Suggested-by: Michal Hocko <mhocko@suse.com> Acked-by: Michal Hocko <mhocko@suse.com> Acked-by: Hillf Danton <hillf.zj@alibaba-inc.com> Cc: Mel Gorman <mgorman@techsingularity.net> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Andrey Ryabinin <aryabinin@virtuozzo.com> Cc: Boris Brezillon <boris.brezillon@free-electrons.com> Cc: Chris Leech <cleech@redhat.com> Cc: "David S. Miller" <davem@davemloft.net> Cc: Eric Dumazet <edumazet@google.com> Cc: Josef Bacik <jbacik@fb.com> Cc: Lee Duncan <lduncan@suse.com> Cc: Michal Hocko <mhocko@suse.com> Cc: Richard Weinberger <richard@nod.at> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'mm')
-rw-r--r--mm/page_alloc.c11
-rw-r--r--mm/vmscan.c17
2 files changed, 17 insertions, 11 deletions
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 1daf509722c7..f9e450c6b6e4 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -3283,15 +3283,15 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
3283 enum compact_priority prio, enum compact_result *compact_result) 3283 enum compact_priority prio, enum compact_result *compact_result)
3284{ 3284{
3285 struct page *page; 3285 struct page *page;
3286 unsigned int noreclaim_flag = current->flags & PF_MEMALLOC; 3286 unsigned int noreclaim_flag;
3287 3287
3288 if (!order) 3288 if (!order)
3289 return NULL; 3289 return NULL;
3290 3290
3291 current->flags |= PF_MEMALLOC; 3291 noreclaim_flag = memalloc_noreclaim_save();
3292 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 3292 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
3293 prio); 3293 prio);
3294 current->flags = (current->flags & ~PF_MEMALLOC) | noreclaim_flag; 3294 memalloc_noreclaim_restore(noreclaim_flag);
3295 3295
3296 if (*compact_result <= COMPACT_INACTIVE) 3296 if (*compact_result <= COMPACT_INACTIVE)
3297 return NULL; 3297 return NULL;
@@ -3438,12 +3438,13 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order,
3438{ 3438{
3439 struct reclaim_state reclaim_state; 3439 struct reclaim_state reclaim_state;
3440 int progress; 3440 int progress;
3441 unsigned int noreclaim_flag;
3441 3442
3442 cond_resched(); 3443 cond_resched();
3443 3444
3444 /* We now go into synchronous reclaim */ 3445 /* We now go into synchronous reclaim */
3445 cpuset_memory_pressure_bump(); 3446 cpuset_memory_pressure_bump();
3446 current->flags |= PF_MEMALLOC; 3447 noreclaim_flag = memalloc_noreclaim_save();
3447 lockdep_set_current_reclaim_state(gfp_mask); 3448 lockdep_set_current_reclaim_state(gfp_mask);
3448 reclaim_state.reclaimed_slab = 0; 3449 reclaim_state.reclaimed_slab = 0;
3449 current->reclaim_state = &reclaim_state; 3450 current->reclaim_state = &reclaim_state;
@@ -3453,7 +3454,7 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order,
3453 3454
3454 current->reclaim_state = NULL; 3455 current->reclaim_state = NULL;
3455 lockdep_clear_current_reclaim_state(); 3456 lockdep_clear_current_reclaim_state();
3456 current->flags &= ~PF_MEMALLOC; 3457 memalloc_noreclaim_restore(noreclaim_flag);
3457 3458
3458 cond_resched(); 3459 cond_resched();
3459 3460
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 4e7ed65842af..2f45c0520f43 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3036,6 +3036,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
3036 struct zonelist *zonelist; 3036 struct zonelist *zonelist;
3037 unsigned long nr_reclaimed; 3037 unsigned long nr_reclaimed;
3038 int nid; 3038 int nid;
3039 unsigned int noreclaim_flag;
3039 struct scan_control sc = { 3040 struct scan_control sc = {
3040 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), 3041 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
3041 .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) | 3042 .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) |
@@ -3062,9 +3063,9 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
3062 sc.gfp_mask, 3063 sc.gfp_mask,
3063 sc.reclaim_idx); 3064 sc.reclaim_idx);
3064 3065
3065 current->flags |= PF_MEMALLOC; 3066 noreclaim_flag = memalloc_noreclaim_save();
3066 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 3067 nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3067 current->flags &= ~PF_MEMALLOC; 3068 memalloc_noreclaim_restore(noreclaim_flag);
3068 3069
3069 trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed); 3070 trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
3070 3071
@@ -3589,8 +3590,9 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
3589 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); 3590 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3590 struct task_struct *p = current; 3591 struct task_struct *p = current;
3591 unsigned long nr_reclaimed; 3592 unsigned long nr_reclaimed;
3593 unsigned int noreclaim_flag;
3592 3594
3593 p->flags |= PF_MEMALLOC; 3595 noreclaim_flag = memalloc_noreclaim_save();
3594 lockdep_set_current_reclaim_state(sc.gfp_mask); 3596 lockdep_set_current_reclaim_state(sc.gfp_mask);
3595 reclaim_state.reclaimed_slab = 0; 3597 reclaim_state.reclaimed_slab = 0;
3596 p->reclaim_state = &reclaim_state; 3598 p->reclaim_state = &reclaim_state;
@@ -3599,7 +3601,7 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
3599 3601
3600 p->reclaim_state = NULL; 3602 p->reclaim_state = NULL;
3601 lockdep_clear_current_reclaim_state(); 3603 lockdep_clear_current_reclaim_state();
3602 p->flags &= ~PF_MEMALLOC; 3604 memalloc_noreclaim_restore(noreclaim_flag);
3603 3605
3604 return nr_reclaimed; 3606 return nr_reclaimed;
3605} 3607}
@@ -3764,6 +3766,7 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
3764 struct task_struct *p = current; 3766 struct task_struct *p = current;
3765 struct reclaim_state reclaim_state; 3767 struct reclaim_state reclaim_state;
3766 int classzone_idx = gfp_zone(gfp_mask); 3768 int classzone_idx = gfp_zone(gfp_mask);
3769 unsigned int noreclaim_flag;
3767 struct scan_control sc = { 3770 struct scan_control sc = {
3768 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), 3771 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
3769 .gfp_mask = (gfp_mask = current_gfp_context(gfp_mask)), 3772 .gfp_mask = (gfp_mask = current_gfp_context(gfp_mask)),
@@ -3781,7 +3784,8 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
3781 * and we also need to be able to write out pages for RECLAIM_WRITE 3784 * and we also need to be able to write out pages for RECLAIM_WRITE
3782 * and RECLAIM_UNMAP. 3785 * and RECLAIM_UNMAP.
3783 */ 3786 */
3784 p->flags |= PF_MEMALLOC | PF_SWAPWRITE; 3787 noreclaim_flag = memalloc_noreclaim_save();
3788 p->flags |= PF_SWAPWRITE;
3785 lockdep_set_current_reclaim_state(gfp_mask); 3789 lockdep_set_current_reclaim_state(gfp_mask);
3786 reclaim_state.reclaimed_slab = 0; 3790 reclaim_state.reclaimed_slab = 0;
3787 p->reclaim_state = &reclaim_state; 3791 p->reclaim_state = &reclaim_state;
@@ -3797,7 +3801,8 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
3797 } 3801 }
3798 3802
3799 p->reclaim_state = NULL; 3803 p->reclaim_state = NULL;
3800 current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE); 3804 current->flags &= ~PF_SWAPWRITE;
3805 memalloc_noreclaim_restore(noreclaim_flag);
3801 lockdep_clear_current_reclaim_state(); 3806 lockdep_clear_current_reclaim_state();
3802 return sc.nr_reclaimed >= nr_pages; 3807 return sc.nr_reclaimed >= nr_pages;
3803} 3808}