aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJens Axboe2009-04-23 05:19:38 -0500
committerJens Axboe2009-04-24 01:54:22 -0500
commitf2d1f0ae7851be5ebd9613a80dac139270938809 (patch)
tree91efc4740f854b94ea84583a2cd6c34d14eafc8e /block/cfq-iosched.c
parent3ac6c9f8a66726745136e46f63600550c3eb6cec (diff)
downloadkernel-common-f2d1f0ae7851be5ebd9613a80dac139270938809.tar.gz
kernel-common-f2d1f0ae7851be5ebd9613a80dac139270938809.tar.xz
kernel-common-f2d1f0ae7851be5ebd9613a80dac139270938809.zip
cfq-iosched: cache prio_tree root in cfqq->p_root
Currently we look it up from ->ioprio, but ->ioprio can change if either the process gets its IO priority changed explicitly, or if cfq decides to temporarily boost it. So if we are unlucky, we can end up attempting to remove a node from a different rbtree root than where it was added. Fix this by using ->org_ioprio as the prio_tree index, since that will only change for explicit IO priority settings (not for a boost). Additionally cache the rbtree root inside the cfqq, then we don't have to add code to reinsert the cfqq in the prio_tree if IO priority changes. Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
Diffstat (limited to 'block/cfq-iosched.c')
-rw-r--r--block/cfq-iosched.c34
1 files changed, 20 insertions, 14 deletions
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index b0b754a6882..a55a9bd75bd 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -154,6 +154,8 @@ struct cfq_queue {
154 unsigned long rb_key; 154 unsigned long rb_key;
155 /* prio tree member */ 155 /* prio tree member */
156 struct rb_node p_node; 156 struct rb_node p_node;
157 /* prio tree root we belong to, if any */
158 struct rb_root *p_root;
157 /* sorted list of pending requests */ 159 /* sorted list of pending requests */
158 struct rb_root sort_list; 160 struct rb_root sort_list;
159 /* if fifo isn't expired, next request to serve */ 161 /* if fifo isn't expired, next request to serve */
@@ -558,10 +560,10 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,
558} 560}
559 561
560static struct cfq_queue * 562static struct cfq_queue *
561cfq_prio_tree_lookup(struct cfq_data *cfqd, int ioprio, sector_t sector, 563cfq_prio_tree_lookup(struct cfq_data *cfqd, struct rb_root *root,
562 struct rb_node **ret_parent, struct rb_node ***rb_link) 564 sector_t sector, struct rb_node **ret_parent,
565 struct rb_node ***rb_link)
563{ 566{
564 struct rb_root *root = &cfqd->prio_trees[ioprio];
565 struct rb_node **p, *parent; 567 struct rb_node **p, *parent;
566 struct cfq_queue *cfqq = NULL; 568 struct cfq_queue *cfqq = NULL;
567 569
@@ -595,24 +597,27 @@ cfq_prio_tree_lookup(struct cfq_data *cfqd, int ioprio, sector_t sector,
595 597
596static void cfq_prio_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq) 598static void cfq_prio_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq)
597{ 599{
598 struct rb_root *root = &cfqd->prio_trees[cfqq->ioprio];
599 struct rb_node **p, *parent; 600 struct rb_node **p, *parent;
600 struct cfq_queue *__cfqq; 601 struct cfq_queue *__cfqq;
601 602
602 if (!RB_EMPTY_NODE(&cfqq->p_node)) 603 if (cfqq->p_root) {
603 rb_erase_init(&cfqq->p_node, root); 604 rb_erase(&cfqq->p_node, cfqq->p_root);
605 cfqq->p_root = NULL;
606 }
604 607
605 if (cfq_class_idle(cfqq)) 608 if (cfq_class_idle(cfqq))
606 return; 609 return;
607 if (!cfqq->next_rq) 610 if (!cfqq->next_rq)
608 return; 611 return;
609 612
610 __cfqq = cfq_prio_tree_lookup(cfqd, cfqq->ioprio, cfqq->next_rq->sector, 613 cfqq->p_root = &cfqd->prio_trees[cfqq->org_ioprio];
614 __cfqq = cfq_prio_tree_lookup(cfqd, cfqq->p_root, cfqq->next_rq->sector,
611 &parent, &p); 615 &parent, &p);
612 if (!__cfqq) { 616 if (!__cfqq) {
613 rb_link_node(&cfqq->p_node, parent, p); 617 rb_link_node(&cfqq->p_node, parent, p);
614 rb_insert_color(&cfqq->p_node, root); 618 rb_insert_color(&cfqq->p_node, cfqq->p_root);
615 } 619 } else
620 cfqq->p_root = NULL;
616} 621}
617 622
618/* 623/*
@@ -657,8 +662,10 @@ static void cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)
657 662
658 if (!RB_EMPTY_NODE(&cfqq->rb_node)) 663 if (!RB_EMPTY_NODE(&cfqq->rb_node))
659 cfq_rb_erase(&cfqq->rb_node, &cfqd->service_tree); 664 cfq_rb_erase(&cfqq->rb_node, &cfqd->service_tree);
660 if (!RB_EMPTY_NODE(&cfqq->p_node)) 665 if (cfqq->p_root) {
661 rb_erase_init(&cfqq->p_node, &cfqd->prio_trees[cfqq->ioprio]); 666 rb_erase(&cfqq->p_node, cfqq->p_root);
667 cfqq->p_root = NULL;
668 }
662 669
663 BUG_ON(!cfqd->busy_queues); 670 BUG_ON(!cfqd->busy_queues);
664 cfqd->busy_queues--; 671 cfqd->busy_queues--;
@@ -965,7 +972,7 @@ static inline int cfq_rq_close(struct cfq_data *cfqd, struct request *rq)
965static struct cfq_queue *cfqq_close(struct cfq_data *cfqd, 972static struct cfq_queue *cfqq_close(struct cfq_data *cfqd,
966 struct cfq_queue *cur_cfqq) 973 struct cfq_queue *cur_cfqq)
967{ 974{
968 struct rb_root *root = &cfqd->prio_trees[cur_cfqq->ioprio]; 975 struct rb_root *root = &cfqd->prio_trees[cur_cfqq->org_ioprio];
969 struct rb_node *parent, *node; 976 struct rb_node *parent, *node;
970 struct cfq_queue *__cfqq; 977 struct cfq_queue *__cfqq;
971 sector_t sector = cfqd->last_position; 978 sector_t sector = cfqd->last_position;
@@ -977,8 +984,7 @@ static struct cfq_queue *cfqq_close(struct cfq_data *cfqd,
977 * First, if we find a request starting at the end of the last 984 * First, if we find a request starting at the end of the last
978 * request, choose it. 985 * request, choose it.
979 */ 986 */
980 __cfqq = cfq_prio_tree_lookup(cfqd, cur_cfqq->ioprio, 987 __cfqq = cfq_prio_tree_lookup(cfqd, root, sector, &parent, NULL);
981 sector, &parent, NULL);
982 if (__cfqq) 988 if (__cfqq)
983 return __cfqq; 989 return __cfqq;
984 990