aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid Woodhouse2018-02-10 17:39:24 -0600
committerGreg Kroah-Hartman2018-02-22 08:44:59 -0600
commitc63497edc3aa9b7e22dc2aa012d26ec4770b8f2d (patch)
tree04790ad89ca1468515ab0b27beb86cc522b8abf1
parent5ff8af891df048c57462cabc42bb03a55a42196e (diff)
downloadkernel-omap-c63497edc3aa9b7e22dc2aa012d26ec4770b8f2d.tar.gz
kernel-omap-c63497edc3aa9b7e22dc2aa012d26ec4770b8f2d.tar.xz
kernel-omap-c63497edc3aa9b7e22dc2aa012d26ec4770b8f2d.zip
KVM/x86: Reduce retpoline performance impact in slot_handle_level_range(), by always inlining iterator helper methods
commit 928a4c39484281f8ca366f53a1db79330d058401 upstream. With retpoline, tight loops of "call this function for every XXX" are very much pessimised by taking a prediction miss *every* time. This one is by far the biggest contributor to the guest launch time with retpoline. By marking the iterator slot_handle_…() functions always_inline, we can ensure that the indirect function call can be optimised away into a direct call and it actually generates slightly smaller code because some of the other conditionals can get optimised away too. Performance is now pretty close to what we see with nospectre_v2 on the command line. Suggested-by: Linus Torvalds <torvalds@linux-foundation.org> Tested-by: Filippo Sironi <sironi@amazon.de> Signed-off-by: David Woodhouse <dwmw@amazon.co.uk> Reviewed-by: Filippo Sironi <sironi@amazon.de> Acked-by: Paolo Bonzini <pbonzini@redhat.com> Cc: Andy Lutomirski <luto@kernel.org> Cc: Arjan van de Ven <arjan@linux.intel.com> Cc: Borislav Petkov <bp@alien8.de> Cc: Dan Williams <dan.j.williams@intel.com> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: David Woodhouse <dwmw2@infradead.org> Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Cc: Josh Poimboeuf <jpoimboe@redhat.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: arjan.van.de.ven@intel.com Cc: dave.hansen@intel.com Cc: jmattson@google.com Cc: karahmed@amazon.de Cc: kvm@vger.kernel.org Cc: rkrcmar@redhat.com Link: http://lkml.kernel.org/r/1518305967-31356-4-git-send-email-dwmw@amazon.co.uk Signed-off-by: Ingo Molnar <mingo@kernel.org> Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
-rw-r--r--arch/x86/kvm/mmu.c10
1 files changed, 5 insertions, 5 deletions
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 1049c3c9b877..2b71f2c03b9e 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -4503,7 +4503,7 @@ void kvm_mmu_setup(struct kvm_vcpu *vcpu)
4503typedef bool (*slot_level_handler) (struct kvm *kvm, unsigned long *rmap); 4503typedef bool (*slot_level_handler) (struct kvm *kvm, unsigned long *rmap);
4504 4504
4505/* The caller should hold mmu-lock before calling this function. */ 4505/* The caller should hold mmu-lock before calling this function. */
4506static bool 4506static __always_inline bool
4507slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot, 4507slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
4508 slot_level_handler fn, int start_level, int end_level, 4508 slot_level_handler fn, int start_level, int end_level,
4509 gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb) 4509 gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
@@ -4533,7 +4533,7 @@ slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
4533 return flush; 4533 return flush;
4534} 4534}
4535 4535
4536static bool 4536static __always_inline bool
4537slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot, 4537slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4538 slot_level_handler fn, int start_level, int end_level, 4538 slot_level_handler fn, int start_level, int end_level,
4539 bool lock_flush_tlb) 4539 bool lock_flush_tlb)
@@ -4544,7 +4544,7 @@ slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4544 lock_flush_tlb); 4544 lock_flush_tlb);
4545} 4545}
4546 4546
4547static bool 4547static __always_inline bool
4548slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot, 4548slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4549 slot_level_handler fn, bool lock_flush_tlb) 4549 slot_level_handler fn, bool lock_flush_tlb)
4550{ 4550{
@@ -4552,7 +4552,7 @@ slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4552 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); 4552 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
4553} 4553}
4554 4554
4555static bool 4555static __always_inline bool
4556slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot, 4556slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4557 slot_level_handler fn, bool lock_flush_tlb) 4557 slot_level_handler fn, bool lock_flush_tlb)
4558{ 4558{
@@ -4560,7 +4560,7 @@ slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
4560 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); 4560 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
4561} 4561}
4562 4562
4563static bool 4563static __always_inline bool
4564slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot, 4564slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
4565 slot_level_handler fn, bool lock_flush_tlb) 4565 slot_level_handler fn, bool lock_flush_tlb)
4566{ 4566{