Revert "HACK: ARM: dts: dra7-ipu-common: Revert to CMA pools for IPU early boots"
[rpmsg/rpmsg.git] / virt / kvm / kvm_main.c
1 /*
2  * Kernel-based Virtual Machine driver for Linux
3  *
4  * This module enables machines with Intel VT-x extensions to run virtual
5  * machines without emulation or binary translation.
6  *
7  * Copyright (C) 2006 Qumranet, Inc.
8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
9  *
10  * Authors:
11  *   Avi Kivity   <avi@qumranet.com>
12  *   Yaniv Kamay  <yaniv@qumranet.com>
13  *
14  * This work is licensed under the terms of the GNU GPL, version 2.  See
15  * the COPYING file in the top-level directory.
16  *
17  */
19 #include <kvm/iodev.h>
21 #include <linux/kvm_host.h>
22 #include <linux/kvm.h>
23 #include <linux/module.h>
24 #include <linux/errno.h>
25 #include <linux/percpu.h>
26 #include <linux/mm.h>
27 #include <linux/miscdevice.h>
28 #include <linux/vmalloc.h>
29 #include <linux/reboot.h>
30 #include <linux/debugfs.h>
31 #include <linux/highmem.h>
32 #include <linux/file.h>
33 #include <linux/syscore_ops.h>
34 #include <linux/cpu.h>
35 #include <linux/sched/signal.h>
36 #include <linux/sched/mm.h>
37 #include <linux/sched/stat.h>
38 #include <linux/cpumask.h>
39 #include <linux/smp.h>
40 #include <linux/anon_inodes.h>
41 #include <linux/profile.h>
42 #include <linux/kvm_para.h>
43 #include <linux/pagemap.h>
44 #include <linux/mman.h>
45 #include <linux/swap.h>
46 #include <linux/bitops.h>
47 #include <linux/spinlock.h>
48 #include <linux/compat.h>
49 #include <linux/srcu.h>
50 #include <linux/hugetlb.h>
51 #include <linux/slab.h>
52 #include <linux/sort.h>
53 #include <linux/bsearch.h>
55 #include <asm/processor.h>
56 #include <asm/io.h>
57 #include <asm/ioctl.h>
58 #include <linux/uaccess.h>
59 #include <asm/pgtable.h>
61 #include "coalesced_mmio.h"
62 #include "async_pf.h"
63 #include "vfio.h"
65 #define CREATE_TRACE_POINTS
66 #include <trace/events/kvm.h>
68 /* Worst case buffer size needed for holding an integer. */
69 #define ITOA_MAX_LEN 12
71 MODULE_AUTHOR("Qumranet");
72 MODULE_LICENSE("GPL");
74 /* Architectures should define their poll value according to the halt latency */
75 unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
76 module_param(halt_poll_ns, uint, 0644);
77 EXPORT_SYMBOL_GPL(halt_poll_ns);
79 /* Default doubles per-vcpu halt_poll_ns. */
80 unsigned int halt_poll_ns_grow = 2;
81 module_param(halt_poll_ns_grow, uint, 0644);
82 EXPORT_SYMBOL_GPL(halt_poll_ns_grow);
84 /* Default resets per-vcpu halt_poll_ns . */
85 unsigned int halt_poll_ns_shrink;
86 module_param(halt_poll_ns_shrink, uint, 0644);
87 EXPORT_SYMBOL_GPL(halt_poll_ns_shrink);
89 /*
90  * Ordering of locks:
91  *
92  *      kvm->lock --> kvm->slots_lock --> kvm->irq_lock
93  */
95 DEFINE_SPINLOCK(kvm_lock);
96 static DEFINE_RAW_SPINLOCK(kvm_count_lock);
97 LIST_HEAD(vm_list);
99 static cpumask_var_t cpus_hardware_enabled;
100 static int kvm_usage_count;
101 static atomic_t hardware_enable_failed;
103 struct kmem_cache *kvm_vcpu_cache;
104 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
106 static __read_mostly struct preempt_ops kvm_preempt_ops;
108 struct dentry *kvm_debugfs_dir;
109 EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
111 static int kvm_debugfs_num_entries;
112 static const struct file_operations *stat_fops_per_vm[];
114 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
115                            unsigned long arg);
116 #ifdef CONFIG_KVM_COMPAT
117 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
118                                   unsigned long arg);
119 #define KVM_COMPAT(c)   .compat_ioctl   = (c)
120 #else
121 static long kvm_no_compat_ioctl(struct file *file, unsigned int ioctl,
122                                 unsigned long arg) { return -EINVAL; }
123 #define KVM_COMPAT(c)   .compat_ioctl   = kvm_no_compat_ioctl
124 #endif
125 static int hardware_enable_all(void);
126 static void hardware_disable_all(void);
128 static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
130 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
132 __visible bool kvm_rebooting;
133 EXPORT_SYMBOL_GPL(kvm_rebooting);
135 static bool largepages_enabled = true;
137 #define KVM_EVENT_CREATE_VM 0
138 #define KVM_EVENT_DESTROY_VM 1
139 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
140 static unsigned long long kvm_createvm_count;
141 static unsigned long long kvm_active_vms;
143 __weak int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
144                 unsigned long start, unsigned long end, bool blockable)
146         return 0;
149 bool kvm_is_reserved_pfn(kvm_pfn_t pfn)
151         if (pfn_valid(pfn))
152                 return PageReserved(pfn_to_page(pfn));
154         return true;
157 /*
158  * Switches to specified vcpu, until a matching vcpu_put()
159  */
160 void vcpu_load(struct kvm_vcpu *vcpu)
162         int cpu = get_cpu();
163         preempt_notifier_register(&vcpu->preempt_notifier);
164         kvm_arch_vcpu_load(vcpu, cpu);
165         put_cpu();
167 EXPORT_SYMBOL_GPL(vcpu_load);
169 void vcpu_put(struct kvm_vcpu *vcpu)
171         preempt_disable();
172         kvm_arch_vcpu_put(vcpu);
173         preempt_notifier_unregister(&vcpu->preempt_notifier);
174         preempt_enable();
176 EXPORT_SYMBOL_GPL(vcpu_put);
178 /* TODO: merge with kvm_arch_vcpu_should_kick */
179 static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req)
181         int mode = kvm_vcpu_exiting_guest_mode(vcpu);
183         /*
184          * We need to wait for the VCPU to reenable interrupts and get out of
185          * READING_SHADOW_PAGE_TABLES mode.
186          */
187         if (req & KVM_REQUEST_WAIT)
188                 return mode != OUTSIDE_GUEST_MODE;
190         /*
191          * Need to kick a running VCPU, but otherwise there is nothing to do.
192          */
193         return mode == IN_GUEST_MODE;
196 static void ack_flush(void *_completed)
200 static inline bool kvm_kick_many_cpus(const struct cpumask *cpus, bool wait)
202         if (unlikely(!cpus))
203                 cpus = cpu_online_mask;
205         if (cpumask_empty(cpus))
206                 return false;
208         smp_call_function_many(cpus, ack_flush, NULL, wait);
209         return true;
212 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
213                                  unsigned long *vcpu_bitmap, cpumask_var_t tmp)
215         int i, cpu, me;
216         struct kvm_vcpu *vcpu;
217         bool called;
219         me = get_cpu();
221         kvm_for_each_vcpu(i, vcpu, kvm) {
222                 if (!test_bit(i, vcpu_bitmap))
223                         continue;
225                 kvm_make_request(req, vcpu);
226                 cpu = vcpu->cpu;
228                 if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu))
229                         continue;
231                 if (tmp != NULL && cpu != -1 && cpu != me &&
232                     kvm_request_needs_ipi(vcpu, req))
233                         __cpumask_set_cpu(cpu, tmp);
234         }
236         called = kvm_kick_many_cpus(tmp, !!(req & KVM_REQUEST_WAIT));
237         put_cpu();
239         return called;
242 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
244         cpumask_var_t cpus;
245         bool called;
246         static unsigned long vcpu_bitmap[BITS_TO_LONGS(KVM_MAX_VCPUS)]
247                 = {[0 ... BITS_TO_LONGS(KVM_MAX_VCPUS)-1] = ULONG_MAX};
249         zalloc_cpumask_var(&cpus, GFP_ATOMIC);
251         called = kvm_make_vcpus_request_mask(kvm, req, vcpu_bitmap, cpus);
253         free_cpumask_var(cpus);
254         return called;
257 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
258 void kvm_flush_remote_tlbs(struct kvm *kvm)
260         /*
261          * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in
262          * kvm_make_all_cpus_request.
263          */
264         long dirty_count = smp_load_acquire(&kvm->tlbs_dirty);
266         /*
267          * We want to publish modifications to the page tables before reading
268          * mode. Pairs with a memory barrier in arch-specific code.
269          * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
270          * and smp_mb in walk_shadow_page_lockless_begin/end.
271          * - powerpc: smp_mb in kvmppc_prepare_to_enter.
272          *
273          * There is already an smp_mb__after_atomic() before
274          * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
275          * barrier here.
276          */
277         if (!kvm_arch_flush_remote_tlb(kvm)
278             || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
279                 ++kvm->stat.remote_tlb_flush;
280         cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
282 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
283 #endif
285 void kvm_reload_remote_mmus(struct kvm *kvm)
287         kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
290 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
292         struct page *page;
293         int r;
295         mutex_init(&vcpu->mutex);
296         vcpu->cpu = -1;
297         vcpu->kvm = kvm;
298         vcpu->vcpu_id = id;
299         vcpu->pid = NULL;
300         init_swait_queue_head(&vcpu->wq);
301         kvm_async_pf_vcpu_init(vcpu);
303         vcpu->pre_pcpu = -1;
304         INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);
306         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
307         if (!page) {
308                 r = -ENOMEM;
309                 goto fail;
310         }
311         vcpu->run = page_address(page);
313         kvm_vcpu_set_in_spin_loop(vcpu, false);
314         kvm_vcpu_set_dy_eligible(vcpu, false);
315         vcpu->preempted = false;
317         r = kvm_arch_vcpu_init(vcpu);
318         if (r < 0)
319                 goto fail_free_run;
320         return 0;
322 fail_free_run:
323         free_page((unsigned long)vcpu->run);
324 fail:
325         return r;
327 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
329 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
331         /*
332          * no need for rcu_read_lock as VCPU_RUN is the only place that
333          * will change the vcpu->pid pointer and on uninit all file
334          * descriptors are already gone.
335          */
336         put_pid(rcu_dereference_protected(vcpu->pid, 1));
337         kvm_arch_vcpu_uninit(vcpu);
338         free_page((unsigned long)vcpu->run);
340 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
342 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
343 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
345         return container_of(mn, struct kvm, mmu_notifier);
348 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
349                                         struct mm_struct *mm,
350                                         unsigned long address,
351                                         pte_t pte)
353         struct kvm *kvm = mmu_notifier_to_kvm(mn);
354         int idx;
356         idx = srcu_read_lock(&kvm->srcu);
357         spin_lock(&kvm->mmu_lock);
358         kvm->mmu_notifier_seq++;
359         kvm_set_spte_hva(kvm, address, pte);
360         spin_unlock(&kvm->mmu_lock);
361         srcu_read_unlock(&kvm->srcu, idx);
364 static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
365                                                     struct mm_struct *mm,
366                                                     unsigned long start,
367                                                     unsigned long end,
368                                                     bool blockable)
370         struct kvm *kvm = mmu_notifier_to_kvm(mn);
371         int need_tlb_flush = 0, idx;
372         int ret;
374         idx = srcu_read_lock(&kvm->srcu);
375         spin_lock(&kvm->mmu_lock);
376         /*
377          * The count increase must become visible at unlock time as no
378          * spte can be established without taking the mmu_lock and
379          * count is also read inside the mmu_lock critical section.
380          */
381         kvm->mmu_notifier_count++;
382         need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
383         need_tlb_flush |= kvm->tlbs_dirty;
384         /* we've to flush the tlb before the pages can be freed */
385         if (need_tlb_flush)
386                 kvm_flush_remote_tlbs(kvm);
388         spin_unlock(&kvm->mmu_lock);
390         ret = kvm_arch_mmu_notifier_invalidate_range(kvm, start, end, blockable);
392         srcu_read_unlock(&kvm->srcu, idx);
394         return ret;
397 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
398                                                   struct mm_struct *mm,
399                                                   unsigned long start,
400                                                   unsigned long end)
402         struct kvm *kvm = mmu_notifier_to_kvm(mn);
404         spin_lock(&kvm->mmu_lock);
405         /*
406          * This sequence increase will notify the kvm page fault that
407          * the page that is going to be mapped in the spte could have
408          * been freed.
409          */
410         kvm->mmu_notifier_seq++;
411         smp_wmb();
412         /*
413          * The above sequence increase must be visible before the
414          * below count decrease, which is ensured by the smp_wmb above
415          * in conjunction with the smp_rmb in mmu_notifier_retry().
416          */
417         kvm->mmu_notifier_count--;
418         spin_unlock(&kvm->mmu_lock);
420         BUG_ON(kvm->mmu_notifier_count < 0);
423 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
424                                               struct mm_struct *mm,
425                                               unsigned long start,
426                                               unsigned long end)
428         struct kvm *kvm = mmu_notifier_to_kvm(mn);
429         int young, idx;
431         idx = srcu_read_lock(&kvm->srcu);
432         spin_lock(&kvm->mmu_lock);
434         young = kvm_age_hva(kvm, start, end);
435         if (young)
436                 kvm_flush_remote_tlbs(kvm);
438         spin_unlock(&kvm->mmu_lock);
439         srcu_read_unlock(&kvm->srcu, idx);
441         return young;
444 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
445                                         struct mm_struct *mm,
446                                         unsigned long start,
447                                         unsigned long end)
449         struct kvm *kvm = mmu_notifier_to_kvm(mn);
450         int young, idx;
452         idx = srcu_read_lock(&kvm->srcu);
453         spin_lock(&kvm->mmu_lock);
454         /*
455          * Even though we do not flush TLB, this will still adversely
456          * affect performance on pre-Haswell Intel EPT, where there is
457          * no EPT Access Bit to clear so that we have to tear down EPT
458          * tables instead. If we find this unacceptable, we can always
459          * add a parameter to kvm_age_hva so that it effectively doesn't
460          * do anything on clear_young.
461          *
462          * Also note that currently we never issue secondary TLB flushes
463          * from clear_young, leaving this job up to the regular system
464          * cadence. If we find this inaccurate, we might come up with a
465          * more sophisticated heuristic later.
466          */
467         young = kvm_age_hva(kvm, start, end);
468         spin_unlock(&kvm->mmu_lock);
469         srcu_read_unlock(&kvm->srcu, idx);
471         return young;
474 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
475                                        struct mm_struct *mm,
476                                        unsigned long address)
478         struct kvm *kvm = mmu_notifier_to_kvm(mn);
479         int young, idx;
481         idx = srcu_read_lock(&kvm->srcu);
482         spin_lock(&kvm->mmu_lock);
483         young = kvm_test_age_hva(kvm, address);
484         spin_unlock(&kvm->mmu_lock);
485         srcu_read_unlock(&kvm->srcu, idx);
487         return young;
490 static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
491                                      struct mm_struct *mm)
493         struct kvm *kvm = mmu_notifier_to_kvm(mn);
494         int idx;
496         idx = srcu_read_lock(&kvm->srcu);
497         kvm_arch_flush_shadow_all(kvm);
498         srcu_read_unlock(&kvm->srcu, idx);
501 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
502         .flags                  = MMU_INVALIDATE_DOES_NOT_BLOCK,
503         .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
504         .invalidate_range_end   = kvm_mmu_notifier_invalidate_range_end,
505         .clear_flush_young      = kvm_mmu_notifier_clear_flush_young,
506         .clear_young            = kvm_mmu_notifier_clear_young,
507         .test_young             = kvm_mmu_notifier_test_young,
508         .change_pte             = kvm_mmu_notifier_change_pte,
509         .release                = kvm_mmu_notifier_release,
510 };
512 static int kvm_init_mmu_notifier(struct kvm *kvm)
514         kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
515         return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
518 #else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
520 static int kvm_init_mmu_notifier(struct kvm *kvm)
522         return 0;
525 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
527 static struct kvm_memslots *kvm_alloc_memslots(void)
529         int i;
530         struct kvm_memslots *slots;
532         slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
533         if (!slots)
534                 return NULL;
536         for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
537                 slots->id_to_index[i] = slots->memslots[i].id = i;
539         return slots;
542 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
544         if (!memslot->dirty_bitmap)
545                 return;
547         kvfree(memslot->dirty_bitmap);
548         memslot->dirty_bitmap = NULL;
551 /*
552  * Free any memory in @free but not in @dont.
553  */
554 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
555                               struct kvm_memory_slot *dont)
557         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
558                 kvm_destroy_dirty_bitmap(free);
560         kvm_arch_free_memslot(kvm, free, dont);
562         free->npages = 0;
565 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
567         struct kvm_memory_slot *memslot;
569         if (!slots)
570                 return;
572         kvm_for_each_memslot(memslot, slots)
573                 kvm_free_memslot(kvm, memslot, NULL);
575         kvfree(slots);
578 static void kvm_destroy_vm_debugfs(struct kvm *kvm)
580         int i;
582         if (!kvm->debugfs_dentry)
583                 return;
585         debugfs_remove_recursive(kvm->debugfs_dentry);
587         if (kvm->debugfs_stat_data) {
588                 for (i = 0; i < kvm_debugfs_num_entries; i++)
589                         kfree(kvm->debugfs_stat_data[i]);
590                 kfree(kvm->debugfs_stat_data);
591         }
594 static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
596         char dir_name[ITOA_MAX_LEN * 2];
597         struct kvm_stat_data *stat_data;
598         struct kvm_stats_debugfs_item *p;
600         if (!debugfs_initialized())
601                 return 0;
603         snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd);
604         kvm->debugfs_dentry = debugfs_create_dir(dir_name, kvm_debugfs_dir);
606         kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
607                                          sizeof(*kvm->debugfs_stat_data),
608                                          GFP_KERNEL);
609         if (!kvm->debugfs_stat_data)
610                 return -ENOMEM;
612         for (p = debugfs_entries; p->name; p++) {
613                 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL);
614                 if (!stat_data)
615                         return -ENOMEM;
617                 stat_data->kvm = kvm;
618                 stat_data->offset = p->offset;
619                 kvm->debugfs_stat_data[p - debugfs_entries] = stat_data;
620                 debugfs_create_file(p->name, 0644, kvm->debugfs_dentry,
621                                     stat_data, stat_fops_per_vm[p->kind]);
622         }
623         return 0;
626 static struct kvm *kvm_create_vm(unsigned long type)
628         int r, i;
629         struct kvm *kvm = kvm_arch_alloc_vm();
631         if (!kvm)
632                 return ERR_PTR(-ENOMEM);
634         spin_lock_init(&kvm->mmu_lock);
635         mmgrab(current->mm);
636         kvm->mm = current->mm;
637         kvm_eventfd_init(kvm);
638         mutex_init(&kvm->lock);
639         mutex_init(&kvm->irq_lock);
640         mutex_init(&kvm->slots_lock);
641         refcount_set(&kvm->users_count, 1);
642         INIT_LIST_HEAD(&kvm->devices);
644         r = kvm_arch_init_vm(kvm, type);
645         if (r)
646                 goto out_err_no_disable;
648         r = hardware_enable_all();
649         if (r)
650                 goto out_err_no_disable;
652 #ifdef CONFIG_HAVE_KVM_IRQFD
653         INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
654 #endif
656         BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
658         r = -ENOMEM;
659         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
660                 struct kvm_memslots *slots = kvm_alloc_memslots();
661                 if (!slots)
662                         goto out_err_no_srcu;
663                 /*
664                  * Generations must be different for each address space.
665                  * Init kvm generation close to the maximum to easily test the
666                  * code of handling generation number wrap-around.
667                  */
668                 slots->generation = i * 2 - 150;
669                 rcu_assign_pointer(kvm->memslots[i], slots);
670         }
672         if (init_srcu_struct(&kvm->srcu))
673                 goto out_err_no_srcu;
674         if (init_srcu_struct(&kvm->irq_srcu))
675                 goto out_err_no_irq_srcu;
676         for (i = 0; i < KVM_NR_BUSES; i++) {
677                 rcu_assign_pointer(kvm->buses[i],
678                         kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL));
679                 if (!kvm->buses[i])
680                         goto out_err;
681         }
683         r = kvm_init_mmu_notifier(kvm);
684         if (r)
685                 goto out_err;
687         spin_lock(&kvm_lock);
688         list_add(&kvm->vm_list, &vm_list);
689         spin_unlock(&kvm_lock);
691         preempt_notifier_inc();
693         return kvm;
695 out_err:
696         cleanup_srcu_struct(&kvm->irq_srcu);
697 out_err_no_irq_srcu:
698         cleanup_srcu_struct(&kvm->srcu);
699 out_err_no_srcu:
700         hardware_disable_all();
701 out_err_no_disable:
702         refcount_set(&kvm->users_count, 0);
703         for (i = 0; i < KVM_NR_BUSES; i++)
704                 kfree(kvm_get_bus(kvm, i));
705         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
706                 kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
707         kvm_arch_free_vm(kvm);
708         mmdrop(current->mm);
709         return ERR_PTR(r);
712 static void kvm_destroy_devices(struct kvm *kvm)
714         struct kvm_device *dev, *tmp;
716         /*
717          * We do not need to take the kvm->lock here, because nobody else
718          * has a reference to the struct kvm at this point and therefore
719          * cannot access the devices list anyhow.
720          */
721         list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
722                 list_del(&dev->vm_node);
723                 dev->ops->destroy(dev);
724         }
727 static void kvm_destroy_vm(struct kvm *kvm)
729         int i;
730         struct mm_struct *mm = kvm->mm;
732         kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);
733         kvm_destroy_vm_debugfs(kvm);
734         kvm_arch_sync_events(kvm);
735         spin_lock(&kvm_lock);
736         list_del(&kvm->vm_list);
737         spin_unlock(&kvm_lock);
738         kvm_free_irq_routing(kvm);
739         for (i = 0; i < KVM_NR_BUSES; i++) {
740                 struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
742                 if (bus)
743                         kvm_io_bus_destroy(bus);
744                 kvm->buses[i] = NULL;
745         }
746         kvm_coalesced_mmio_free(kvm);
747 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
748         mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
749 #else
750         kvm_arch_flush_shadow_all(kvm);
751 #endif
752         kvm_arch_destroy_vm(kvm);
753         kvm_destroy_devices(kvm);
754         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
755                 kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
756         cleanup_srcu_struct(&kvm->irq_srcu);
757         cleanup_srcu_struct(&kvm->srcu);
758         kvm_arch_free_vm(kvm);
759         preempt_notifier_dec();
760         hardware_disable_all();
761         mmdrop(mm);
764 void kvm_get_kvm(struct kvm *kvm)
766         refcount_inc(&kvm->users_count);
768 EXPORT_SYMBOL_GPL(kvm_get_kvm);
770 void kvm_put_kvm(struct kvm *kvm)
772         if (refcount_dec_and_test(&kvm->users_count))
773                 kvm_destroy_vm(kvm);
775 EXPORT_SYMBOL_GPL(kvm_put_kvm);
778 static int kvm_vm_release(struct inode *inode, struct file *filp)
780         struct kvm *kvm = filp->private_data;
782         kvm_irqfd_release(kvm);
784         kvm_put_kvm(kvm);
785         return 0;
788 /*
789  * Allocation size is twice as large as the actual dirty bitmap size.
790  * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
791  */
792 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
794         unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
796         memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL);
797         if (!memslot->dirty_bitmap)
798                 return -ENOMEM;
800         return 0;
803 /*
804  * Insert memslot and re-sort memslots based on their GFN,
805  * so binary search could be used to lookup GFN.
806  * Sorting algorithm takes advantage of having initially
807  * sorted array and known changed memslot position.
808  */
809 static void update_memslots(struct kvm_memslots *slots,
810                             struct kvm_memory_slot *new)
812         int id = new->id;
813         int i = slots->id_to_index[id];
814         struct kvm_memory_slot *mslots = slots->memslots;
816         WARN_ON(mslots[i].id != id);
817         if (!new->npages) {
818                 WARN_ON(!mslots[i].npages);
819                 if (mslots[i].npages)
820                         slots->used_slots--;
821         } else {
822                 if (!mslots[i].npages)
823                         slots->used_slots++;
824         }
826         while (i < KVM_MEM_SLOTS_NUM - 1 &&
827                new->base_gfn <= mslots[i + 1].base_gfn) {
828                 if (!mslots[i + 1].npages)
829                         break;
830                 mslots[i] = mslots[i + 1];
831                 slots->id_to_index[mslots[i].id] = i;
832                 i++;
833         }
835         /*
836          * The ">=" is needed when creating a slot with base_gfn == 0,
837          * so that it moves before all those with base_gfn == npages == 0.
838          *
839          * On the other hand, if new->npages is zero, the above loop has
840          * already left i pointing to the beginning of the empty part of
841          * mslots, and the ">=" would move the hole backwards in this
842          * case---which is wrong.  So skip the loop when deleting a slot.
843          */
844         if (new->npages) {
845                 while (i > 0 &&
846                        new->base_gfn >= mslots[i - 1].base_gfn) {
847                         mslots[i] = mslots[i - 1];
848                         slots->id_to_index[mslots[i].id] = i;
849                         i--;
850                 }
851         } else
852                 WARN_ON_ONCE(i != slots->used_slots);
854         mslots[i] = *new;
855         slots->id_to_index[mslots[i].id] = i;
858 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
860         u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
862 #ifdef __KVM_HAVE_READONLY_MEM
863         valid_flags |= KVM_MEM_READONLY;
864 #endif
866         if (mem->flags & ~valid_flags)
867                 return -EINVAL;
869         return 0;
872 static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
873                 int as_id, struct kvm_memslots *slots)
875         struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
877         /*
878          * Set the low bit in the generation, which disables SPTE caching
879          * until the end of synchronize_srcu_expedited.
880          */
881         WARN_ON(old_memslots->generation & 1);
882         slots->generation = old_memslots->generation + 1;
884         rcu_assign_pointer(kvm->memslots[as_id], slots);
885         synchronize_srcu_expedited(&kvm->srcu);
887         /*
888          * Increment the new memslot generation a second time. This prevents
889          * vm exits that race with memslot updates from caching a memslot
890          * generation that will (potentially) be valid forever.
891          *
892          * Generations must be unique even across address spaces.  We do not need
893          * a global counter for that, instead the generation space is evenly split
894          * across address spaces.  For example, with two address spaces, address
895          * space 0 will use generations 0, 4, 8, ... while * address space 1 will
896          * use generations 2, 6, 10, 14, ...
897          */
898         slots->generation += KVM_ADDRESS_SPACE_NUM * 2 - 1;
900         kvm_arch_memslots_updated(kvm, slots);
902         return old_memslots;
905 /*
906  * Allocate some memory and give it an address in the guest physical address
907  * space.
908  *
909  * Discontiguous memory is allowed, mostly for framebuffers.
910  *
911  * Must be called holding kvm->slots_lock for write.
912  */
913 int __kvm_set_memory_region(struct kvm *kvm,
914                             const struct kvm_userspace_memory_region *mem)
916         int r;
917         gfn_t base_gfn;
918         unsigned long npages;
919         struct kvm_memory_slot *slot;
920         struct kvm_memory_slot old, new;
921         struct kvm_memslots *slots = NULL, *old_memslots;
922         int as_id, id;
923         enum kvm_mr_change change;
925         r = check_memory_region_flags(mem);
926         if (r)
927                 goto out;
929         r = -EINVAL;
930         as_id = mem->slot >> 16;
931         id = (u16)mem->slot;
933         /* General sanity checks */
934         if (mem->memory_size & (PAGE_SIZE - 1))
935                 goto out;
936         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
937                 goto out;
938         /* We can read the guest memory with __xxx_user() later on. */
939         if ((id < KVM_USER_MEM_SLOTS) &&
940             ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
941              !access_ok(VERIFY_WRITE,
942                         (void __user *)(unsigned long)mem->userspace_addr,
943                         mem->memory_size)))
944                 goto out;
945         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
946                 goto out;
947         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
948                 goto out;
950         slot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
951         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
952         npages = mem->memory_size >> PAGE_SHIFT;
954         if (npages > KVM_MEM_MAX_NR_PAGES)
955                 goto out;
957         new = old = *slot;
959         new.id = id;
960         new.base_gfn = base_gfn;
961         new.npages = npages;
962         new.flags = mem->flags;
964         if (npages) {
965                 if (!old.npages)
966                         change = KVM_MR_CREATE;
967                 else { /* Modify an existing slot. */
968                         if ((mem->userspace_addr != old.userspace_addr) ||
969                             (npages != old.npages) ||
970                             ((new.flags ^ old.flags) & KVM_MEM_READONLY))
971                                 goto out;
973                         if (base_gfn != old.base_gfn)
974                                 change = KVM_MR_MOVE;
975                         else if (new.flags != old.flags)
976                                 change = KVM_MR_FLAGS_ONLY;
977                         else { /* Nothing to change. */
978                                 r = 0;
979                                 goto out;
980                         }
981                 }
982         } else {
983                 if (!old.npages)
984                         goto out;
986                 change = KVM_MR_DELETE;
987                 new.base_gfn = 0;
988                 new.flags = 0;
989         }
991         if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
992                 /* Check for overlaps */
993                 r = -EEXIST;
994                 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) {
995                         if (slot->id == id)
996                                 continue;
997                         if (!((base_gfn + npages <= slot->base_gfn) ||
998                               (base_gfn >= slot->base_gfn + slot->npages)))
999                                 goto out;
1000                 }
1001         }
1003         /* Free page dirty bitmap if unneeded */
1004         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
1005                 new.dirty_bitmap = NULL;
1007         r = -ENOMEM;
1008         if (change == KVM_MR_CREATE) {
1009                 new.userspace_addr = mem->userspace_addr;
1011                 if (kvm_arch_create_memslot(kvm, &new, npages))
1012                         goto out_free;
1013         }
1015         /* Allocate page dirty bitmap if needed */
1016         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
1017                 if (kvm_create_dirty_bitmap(&new) < 0)
1018                         goto out_free;
1019         }
1021         slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
1022         if (!slots)
1023                 goto out_free;
1024         memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots));
1026         if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
1027                 slot = id_to_memslot(slots, id);
1028                 slot->flags |= KVM_MEMSLOT_INVALID;
1030                 old_memslots = install_new_memslots(kvm, as_id, slots);
1032                 /* From this point no new shadow pages pointing to a deleted,
1033                  * or moved, memslot will be created.
1034                  *
1035                  * validation of sp->gfn happens in:
1036                  *      - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
1037                  *      - kvm_is_visible_gfn (mmu_check_roots)
1038                  */
1039                 kvm_arch_flush_shadow_memslot(kvm, slot);
1041                 /*
1042                  * We can re-use the old_memslots from above, the only difference
1043                  * from the currently installed memslots is the invalid flag.  This
1044                  * will get overwritten by update_memslots anyway.
1045                  */
1046                 slots = old_memslots;
1047         }
1049         r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
1050         if (r)
1051                 goto out_slots;
1053         /* actual memory is freed via old in kvm_free_memslot below */
1054         if (change == KVM_MR_DELETE) {
1055                 new.dirty_bitmap = NULL;
1056                 memset(&new.arch, 0, sizeof(new.arch));
1057         }
1059         update_memslots(slots, &new);
1060         old_memslots = install_new_memslots(kvm, as_id, slots);
1062         kvm_arch_commit_memory_region(kvm, mem, &old, &new, change);
1064         kvm_free_memslot(kvm, &old, &new);
1065         kvfree(old_memslots);
1066         return 0;
1068 out_slots:
1069         kvfree(slots);
1070 out_free:
1071         kvm_free_memslot(kvm, &new, &old);
1072 out:
1073         return r;
1075 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
1077 int kvm_set_memory_region(struct kvm *kvm,
1078                           const struct kvm_userspace_memory_region *mem)
1080         int r;
1082         mutex_lock(&kvm->slots_lock);
1083         r = __kvm_set_memory_region(kvm, mem);
1084         mutex_unlock(&kvm->slots_lock);
1085         return r;
1087 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
1089 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
1090                                           struct kvm_userspace_memory_region *mem)
1092         if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
1093                 return -EINVAL;
1095         return kvm_set_memory_region(kvm, mem);
1098 int kvm_get_dirty_log(struct kvm *kvm,
1099                         struct kvm_dirty_log *log, int *is_dirty)
1101         struct kvm_memslots *slots;
1102         struct kvm_memory_slot *memslot;
1103         int i, as_id, id;
1104         unsigned long n;
1105         unsigned long any = 0;
1107         as_id = log->slot >> 16;
1108         id = (u16)log->slot;
1109         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1110                 return -EINVAL;
1112         slots = __kvm_memslots(kvm, as_id);
1113         memslot = id_to_memslot(slots, id);
1114         if (!memslot->dirty_bitmap)
1115                 return -ENOENT;
1117         n = kvm_dirty_bitmap_bytes(memslot);
1119         for (i = 0; !any && i < n/sizeof(long); ++i)
1120                 any = memslot->dirty_bitmap[i];
1122         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
1123                 return -EFAULT;
1125         if (any)
1126                 *is_dirty = 1;
1127         return 0;
1129 EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
1131 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
1132 /**
1133  * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages
1134  *      are dirty write protect them for next write.
1135  * @kvm:        pointer to kvm instance
1136  * @log:        slot id and address to which we copy the log
1137  * @is_dirty:   flag set if any page is dirty
1138  *
1139  * We need to keep it in mind that VCPU threads can write to the bitmap
1140  * concurrently. So, to avoid losing track of dirty pages we keep the
1141  * following order:
1142  *
1143  *    1. Take a snapshot of the bit and clear it if needed.
1144  *    2. Write protect the corresponding page.
1145  *    3. Copy the snapshot to the userspace.
1146  *    4. Upon return caller flushes TLB's if needed.
1147  *
1148  * Between 2 and 4, the guest may write to the page using the remaining TLB
1149  * entry.  This is not a problem because the page is reported dirty using
1150  * the snapshot taken before and step 4 ensures that writes done after
1151  * exiting to userspace will be logged for the next call.
1152  *
1153  */
1154 int kvm_get_dirty_log_protect(struct kvm *kvm,
1155                         struct kvm_dirty_log *log, bool *is_dirty)
1157         struct kvm_memslots *slots;
1158         struct kvm_memory_slot *memslot;
1159         int i, as_id, id;
1160         unsigned long n;
1161         unsigned long *dirty_bitmap;
1162         unsigned long *dirty_bitmap_buffer;
1164         as_id = log->slot >> 16;
1165         id = (u16)log->slot;
1166         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1167                 return -EINVAL;
1169         slots = __kvm_memslots(kvm, as_id);
1170         memslot = id_to_memslot(slots, id);
1172         dirty_bitmap = memslot->dirty_bitmap;
1173         if (!dirty_bitmap)
1174                 return -ENOENT;
1176         n = kvm_dirty_bitmap_bytes(memslot);
1178         dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
1179         memset(dirty_bitmap_buffer, 0, n);
1181         spin_lock(&kvm->mmu_lock);
1182         *is_dirty = false;
1183         for (i = 0; i < n / sizeof(long); i++) {
1184                 unsigned long mask;
1185                 gfn_t offset;
1187                 if (!dirty_bitmap[i])
1188                         continue;
1190                 *is_dirty = true;
1192                 mask = xchg(&dirty_bitmap[i], 0);
1193                 dirty_bitmap_buffer[i] = mask;
1195                 if (mask) {
1196                         offset = i * BITS_PER_LONG;
1197                         kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1198                                                                 offset, mask);
1199                 }
1200         }
1202         spin_unlock(&kvm->mmu_lock);
1203         if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
1204                 return -EFAULT;
1205         return 0;
1207 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
1208 #endif
1210 bool kvm_largepages_enabled(void)
1212         return largepages_enabled;
1215 void kvm_disable_largepages(void)
1217         largepages_enabled = false;
1219 EXPORT_SYMBOL_GPL(kvm_disable_largepages);
1221 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
1223         return __gfn_to_memslot(kvm_memslots(kvm), gfn);
1225 EXPORT_SYMBOL_GPL(gfn_to_memslot);
1227 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
1229         return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
1232 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
1234         struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1236         if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1237               memslot->flags & KVM_MEMSLOT_INVALID)
1238                 return false;
1240         return true;
1242 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
1244 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
1246         struct vm_area_struct *vma;
1247         unsigned long addr, size;
1249         size = PAGE_SIZE;
1251         addr = gfn_to_hva(kvm, gfn);
1252         if (kvm_is_error_hva(addr))
1253                 return PAGE_SIZE;
1255         down_read(&current->mm->mmap_sem);
1256         vma = find_vma(current->mm, addr);
1257         if (!vma)
1258                 goto out;
1260         size = vma_kernel_pagesize(vma);
1262 out:
1263         up_read(&current->mm->mmap_sem);
1265         return size;
1268 static bool memslot_is_readonly(struct kvm_memory_slot *slot)
1270         return slot->flags & KVM_MEM_READONLY;
1273 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1274                                        gfn_t *nr_pages, bool write)
1276         if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
1277                 return KVM_HVA_ERR_BAD;
1279         if (memslot_is_readonly(slot) && write)
1280                 return KVM_HVA_ERR_RO_BAD;
1282         if (nr_pages)
1283                 *nr_pages = slot->npages - (gfn - slot->base_gfn);
1285         return __gfn_to_hva_memslot(slot, gfn);
1288 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1289                                      gfn_t *nr_pages)
1291         return __gfn_to_hva_many(slot, gfn, nr_pages, true);
1294 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1295                                         gfn_t gfn)
1297         return gfn_to_hva_many(slot, gfn, NULL);
1299 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
1301 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1303         return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1305 EXPORT_SYMBOL_GPL(gfn_to_hva);
1307 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
1309         return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
1311 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);
1313 /*
1314  * If writable is set to false, the hva returned by this function is only
1315  * allowed to be read.
1316  */
1317 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
1318                                       gfn_t gfn, bool *writable)
1320         unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
1322         if (!kvm_is_error_hva(hva) && writable)
1323                 *writable = !memslot_is_readonly(slot);
1325         return hva;
1328 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
1330         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1332         return gfn_to_hva_memslot_prot(slot, gfn, writable);
1335 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
1337         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1339         return gfn_to_hva_memslot_prot(slot, gfn, writable);
1342 static inline int check_user_page_hwpoison(unsigned long addr)
1344         int rc, flags = FOLL_HWPOISON | FOLL_WRITE;
1346         rc = get_user_pages(addr, 1, flags, NULL, NULL);
1347         return rc == -EHWPOISON;
1350 /*
1351  * The fast path to get the writable pfn which will be stored in @pfn,
1352  * true indicates success, otherwise false is returned.  It's also the
1353  * only part that runs if we can are in atomic context.
1354  */
1355 static bool hva_to_pfn_fast(unsigned long addr, bool write_fault,
1356                             bool *writable, kvm_pfn_t *pfn)
1358         struct page *page[1];
1359         int npages;
1361         /*
1362          * Fast pin a writable pfn only if it is a write fault request
1363          * or the caller allows to map a writable pfn for a read fault
1364          * request.
1365          */
1366         if (!(write_fault || writable))
1367                 return false;
1369         npages = __get_user_pages_fast(addr, 1, 1, page);
1370         if (npages == 1) {
1371                 *pfn = page_to_pfn(page[0]);
1373                 if (writable)
1374                         *writable = true;
1375                 return true;
1376         }
1378         return false;
1381 /*
1382  * The slow path to get the pfn of the specified host virtual address,
1383  * 1 indicates success, -errno is returned if error is detected.
1384  */
1385 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
1386                            bool *writable, kvm_pfn_t *pfn)
1388         unsigned int flags = FOLL_HWPOISON;
1389         struct page *page;
1390         int npages = 0;
1392         might_sleep();
1394         if (writable)
1395                 *writable = write_fault;
1397         if (write_fault)
1398                 flags |= FOLL_WRITE;
1399         if (async)
1400                 flags |= FOLL_NOWAIT;
1402         npages = get_user_pages_unlocked(addr, 1, &page, flags);
1403         if (npages != 1)
1404                 return npages;
1406         /* map read fault as writable if possible */
1407         if (unlikely(!write_fault) && writable) {
1408                 struct page *wpage;
1410                 if (__get_user_pages_fast(addr, 1, 1, &wpage) == 1) {
1411                         *writable = true;
1412                         put_page(page);
1413                         page = wpage;
1414                 }
1415         }
1416         *pfn = page_to_pfn(page);
1417         return npages;
1420 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
1422         if (unlikely(!(vma->vm_flags & VM_READ)))
1423                 return false;
1425         if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
1426                 return false;
1428         return true;
1431 static int hva_to_pfn_remapped(struct vm_area_struct *vma,
1432                                unsigned long addr, bool *async,
1433                                bool write_fault, bool *writable,
1434                                kvm_pfn_t *p_pfn)
1436         unsigned long pfn;
1437         int r;
1439         r = follow_pfn(vma, addr, &pfn);
1440         if (r) {
1441                 /*
1442                  * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
1443                  * not call the fault handler, so do it here.
1444                  */
1445                 bool unlocked = false;
1446                 r = fixup_user_fault(current, current->mm, addr,
1447                                      (write_fault ? FAULT_FLAG_WRITE : 0),
1448                                      &unlocked);
1449                 if (unlocked)
1450                         return -EAGAIN;
1451                 if (r)
1452                         return r;
1454                 r = follow_pfn(vma, addr, &pfn);
1455                 if (r)
1456                         return r;
1458         }
1460         if (writable)
1461                 *writable = true;
1463         /*
1464          * Get a reference here because callers of *hva_to_pfn* and
1465          * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the
1466          * returned pfn.  This is only needed if the VMA has VM_MIXEDMAP
1467          * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will
1468          * simply do nothing for reserved pfns.
1469          *
1470          * Whoever called remap_pfn_range is also going to call e.g.
1471          * unmap_mapping_range before the underlying pages are freed,
1472          * causing a call to our MMU notifier.
1473          */ 
1474         kvm_get_pfn(pfn);
1476         *p_pfn = pfn;
1477         return 0;
1480 /*
1481  * Pin guest page in memory and return its pfn.
1482  * @addr: host virtual address which maps memory to the guest
1483  * @atomic: whether this function can sleep
1484  * @async: whether this function need to wait IO complete if the
1485  *         host page is not in the memory
1486  * @write_fault: whether we should get a writable host page
1487  * @writable: whether it allows to map a writable host page for !@write_fault
1488  *
1489  * The function will map a writable host page for these two cases:
1490  * 1): @write_fault = true
1491  * 2): @write_fault = false && @writable, @writable will tell the caller
1492  *     whether the mapping is writable.
1493  */
1494 static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
1495                         bool write_fault, bool *writable)
1497         struct vm_area_struct *vma;
1498         kvm_pfn_t pfn = 0;
1499         int npages, r;
1501         /* we can do it either atomically or asynchronously, not both */
1502         BUG_ON(atomic && async);
1504         if (hva_to_pfn_fast(addr, write_fault, writable, &pfn))
1505                 return pfn;
1507         if (atomic)
1508                 return KVM_PFN_ERR_FAULT;
1510         npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
1511         if (npages == 1)
1512                 return pfn;
1514         down_read(&current->mm->mmap_sem);
1515         if (npages == -EHWPOISON ||
1516               (!async && check_user_page_hwpoison(addr))) {
1517                 pfn = KVM_PFN_ERR_HWPOISON;
1518                 goto exit;
1519         }
1521 retry:
1522         vma = find_vma_intersection(current->mm, addr, addr + 1);
1524         if (vma == NULL)
1525                 pfn = KVM_PFN_ERR_FAULT;
1526         else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
1527                 r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable, &pfn);
1528                 if (r == -EAGAIN)
1529                         goto retry;
1530                 if (r < 0)
1531                         pfn = KVM_PFN_ERR_FAULT;
1532         } else {
1533                 if (async && vma_is_valid(vma, write_fault))
1534                         *async = true;
1535                 pfn = KVM_PFN_ERR_FAULT;
1536         }
1537 exit:
1538         up_read(&current->mm->mmap_sem);
1539         return pfn;
1542 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn,
1543                                bool atomic, bool *async, bool write_fault,
1544                                bool *writable)
1546         unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
1548         if (addr == KVM_HVA_ERR_RO_BAD) {
1549                 if (writable)
1550                         *writable = false;
1551                 return KVM_PFN_ERR_RO_FAULT;
1552         }
1554         if (kvm_is_error_hva(addr)) {
1555                 if (writable)
1556                         *writable = false;
1557                 return KVM_PFN_NOSLOT;
1558         }
1560         /* Do not map writable pfn in the readonly memslot. */
1561         if (writable && memslot_is_readonly(slot)) {
1562                 *writable = false;
1563                 writable = NULL;
1564         }
1566         return hva_to_pfn(addr, atomic, async, write_fault,
1567                           writable);
1569 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot);
1571 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1572                       bool *writable)
1574         return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,
1575                                     write_fault, writable);
1577 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1579 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1581         return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1583 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot);
1585 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1587         return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1589 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1591 kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1593         return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn);
1595 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1597 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)
1599         return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1601 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic);
1603 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
1605         return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
1607 EXPORT_SYMBOL_GPL(gfn_to_pfn);
1609 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
1611         return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1613 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn);
1615 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1616                             struct page **pages, int nr_pages)
1618         unsigned long addr;
1619         gfn_t entry = 0;
1621         addr = gfn_to_hva_many(slot, gfn, &entry);
1622         if (kvm_is_error_hva(addr))
1623                 return -1;
1625         if (entry < nr_pages)
1626                 return 0;
1628         return __get_user_pages_fast(addr, nr_pages, 1, pages);
1630 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1632 static struct page *kvm_pfn_to_page(kvm_pfn_t pfn)
1634         if (is_error_noslot_pfn(pfn))
1635                 return KVM_ERR_PTR_BAD_PAGE;
1637         if (kvm_is_reserved_pfn(pfn)) {
1638                 WARN_ON(1);
1639                 return KVM_ERR_PTR_BAD_PAGE;
1640         }
1642         return pfn_to_page(pfn);
1645 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1647         kvm_pfn_t pfn;
1649         pfn = gfn_to_pfn(kvm, gfn);
1651         return kvm_pfn_to_page(pfn);
1653 EXPORT_SYMBOL_GPL(gfn_to_page);
1655 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)
1657         kvm_pfn_t pfn;
1659         pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);
1661         return kvm_pfn_to_page(pfn);
1663 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page);
1665 void kvm_release_page_clean(struct page *page)
1667         WARN_ON(is_error_page(page));
1669         kvm_release_pfn_clean(page_to_pfn(page));
1671 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1673 void kvm_release_pfn_clean(kvm_pfn_t pfn)
1675         if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
1676                 put_page(pfn_to_page(pfn));
1678 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1680 void kvm_release_page_dirty(struct page *page)
1682         WARN_ON(is_error_page(page));
1684         kvm_release_pfn_dirty(page_to_pfn(page));
1686 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1688 void kvm_release_pfn_dirty(kvm_pfn_t pfn)
1690         kvm_set_pfn_dirty(pfn);
1691         kvm_release_pfn_clean(pfn);
1693 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
1695 void kvm_set_pfn_dirty(kvm_pfn_t pfn)
1697         if (!kvm_is_reserved_pfn(pfn)) {
1698                 struct page *page = pfn_to_page(pfn);
1700                 if (!PageReserved(page))
1701                         SetPageDirty(page);
1702         }
1704 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1706 void kvm_set_pfn_accessed(kvm_pfn_t pfn)
1708         if (!kvm_is_reserved_pfn(pfn))
1709                 mark_page_accessed(pfn_to_page(pfn));
1711 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1713 void kvm_get_pfn(kvm_pfn_t pfn)
1715         if (!kvm_is_reserved_pfn(pfn))
1716                 get_page(pfn_to_page(pfn));
1718 EXPORT_SYMBOL_GPL(kvm_get_pfn);
1720 static int next_segment(unsigned long len, int offset)
1722         if (len > PAGE_SIZE - offset)
1723                 return PAGE_SIZE - offset;
1724         else
1725                 return len;
1728 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
1729                                  void *data, int offset, int len)
1731         int r;
1732         unsigned long addr;
1734         addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1735         if (kvm_is_error_hva(addr))
1736                 return -EFAULT;
1737         r = __copy_from_user(data, (void __user *)addr + offset, len);
1738         if (r)
1739                 return -EFAULT;
1740         return 0;
1743 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1744                         int len)
1746         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1748         return __kvm_read_guest_page(slot, gfn, data, offset, len);
1750 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1752 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
1753                              int offset, int len)
1755         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1757         return __kvm_read_guest_page(slot, gfn, data, offset, len);
1759 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);
1761 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1763         gfn_t gfn = gpa >> PAGE_SHIFT;
1764         int seg;
1765         int offset = offset_in_page(gpa);
1766         int ret;
1768         while ((seg = next_segment(len, offset)) != 0) {
1769                 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1770                 if (ret < 0)
1771                         return ret;
1772                 offset = 0;
1773                 len -= seg;
1774                 data += seg;
1775                 ++gfn;
1776         }
1777         return 0;
1779 EXPORT_SYMBOL_GPL(kvm_read_guest);
1781 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
1783         gfn_t gfn = gpa >> PAGE_SHIFT;
1784         int seg;
1785         int offset = offset_in_page(gpa);
1786         int ret;
1788         while ((seg = next_segment(len, offset)) != 0) {
1789                 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
1790                 if (ret < 0)
1791                         return ret;
1792                 offset = 0;
1793                 len -= seg;
1794                 data += seg;
1795                 ++gfn;
1796         }
1797         return 0;
1799 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
1801 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1802                                    void *data, int offset, unsigned long len)
1804         int r;
1805         unsigned long addr;
1807         addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1808         if (kvm_is_error_hva(addr))
1809                 return -EFAULT;
1810         pagefault_disable();
1811         r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1812         pagefault_enable();
1813         if (r)
1814                 return -EFAULT;
1815         return 0;
1818 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1819                           unsigned long len)
1821         gfn_t gfn = gpa >> PAGE_SHIFT;
1822         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1823         int offset = offset_in_page(gpa);
1825         return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1827 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic);
1829 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
1830                                void *data, unsigned long len)
1832         gfn_t gfn = gpa >> PAGE_SHIFT;
1833         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1834         int offset = offset_in_page(gpa);
1836         return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1838 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);
1840 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,
1841                                   const void *data, int offset, int len)
1843         int r;
1844         unsigned long addr;
1846         addr = gfn_to_hva_memslot(memslot, gfn);
1847         if (kvm_is_error_hva(addr))
1848                 return -EFAULT;
1849         r = __copy_to_user((void __user *)addr + offset, data, len);
1850         if (r)
1851                 return -EFAULT;
1852         mark_page_dirty_in_slot(memslot, gfn);
1853         return 0;
1856 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
1857                          const void *data, int offset, int len)
1859         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1861         return __kvm_write_guest_page(slot, gfn, data, offset, len);
1863 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1865 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
1866                               const void *data, int offset, int len)
1868         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1870         return __kvm_write_guest_page(slot, gfn, data, offset, len);
1872 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);
1874 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1875                     unsigned long len)
1877         gfn_t gfn = gpa >> PAGE_SHIFT;
1878         int seg;
1879         int offset = offset_in_page(gpa);
1880         int ret;
1882         while ((seg = next_segment(len, offset)) != 0) {
1883                 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1884                 if (ret < 0)
1885                         return ret;
1886                 offset = 0;
1887                 len -= seg;
1888                 data += seg;
1889                 ++gfn;
1890         }
1891         return 0;
1893 EXPORT_SYMBOL_GPL(kvm_write_guest);
1895 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
1896                          unsigned long len)
1898         gfn_t gfn = gpa >> PAGE_SHIFT;
1899         int seg;
1900         int offset = offset_in_page(gpa);
1901         int ret;
1903         while ((seg = next_segment(len, offset)) != 0) {
1904                 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
1905                 if (ret < 0)
1906                         return ret;
1907                 offset = 0;
1908                 len -= seg;
1909                 data += seg;
1910                 ++gfn;
1911         }
1912         return 0;
1914 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);
1916 static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots,
1917                                        struct gfn_to_hva_cache *ghc,
1918                                        gpa_t gpa, unsigned long len)
1920         int offset = offset_in_page(gpa);
1921         gfn_t start_gfn = gpa >> PAGE_SHIFT;
1922         gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
1923         gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
1924         gfn_t nr_pages_avail;
1926         ghc->gpa = gpa;
1927         ghc->generation = slots->generation;
1928         ghc->len = len;
1929         ghc->memslot = __gfn_to_memslot(slots, start_gfn);
1930         ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL);
1931         if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) {
1932                 ghc->hva += offset;
1933         } else {
1934                 /*
1935                  * If the requested region crosses two memslots, we still
1936                  * verify that the entire region is valid here.
1937                  */
1938                 while (start_gfn <= end_gfn) {
1939                         nr_pages_avail = 0;
1940                         ghc->memslot = __gfn_to_memslot(slots, start_gfn);
1941                         ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
1942                                                    &nr_pages_avail);
1943                         if (kvm_is_error_hva(ghc->hva))
1944                                 return -EFAULT;
1945                         start_gfn += nr_pages_avail;
1946                 }
1947                 /* Use the slow path for cross page reads and writes. */
1948                 ghc->memslot = NULL;
1949         }
1950         return 0;
1953 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1954                               gpa_t gpa, unsigned long len)
1956         struct kvm_memslots *slots = kvm_memslots(kvm);
1957         return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len);
1959 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
1961 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1962                            void *data, int offset, unsigned long len)
1964         struct kvm_memslots *slots = kvm_memslots(kvm);
1965         int r;
1966         gpa_t gpa = ghc->gpa + offset;
1968         BUG_ON(len + offset > ghc->len);
1970         if (slots->generation != ghc->generation)
1971                 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
1973         if (unlikely(!ghc->memslot))
1974                 return kvm_write_guest(kvm, gpa, data, len);
1976         if (kvm_is_error_hva(ghc->hva))
1977                 return -EFAULT;
1979         r = __copy_to_user((void __user *)ghc->hva + offset, data, len);
1980         if (r)
1981                 return -EFAULT;
1982         mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT);
1984         return 0;
1986 EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached);
1988 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1989                            void *data, unsigned long len)
1991         return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);
1993 EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
1995 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1996                            void *data, unsigned long len)
1998         struct kvm_memslots *slots = kvm_memslots(kvm);
1999         int r;
2001         BUG_ON(len > ghc->len);
2003         if (slots->generation != ghc->generation)
2004                 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
2006         if (unlikely(!ghc->memslot))
2007                 return kvm_read_guest(kvm, ghc->gpa, data, len);
2009         if (kvm_is_error_hva(ghc->hva))
2010                 return -EFAULT;
2012         r = __copy_from_user(data, (void __user *)ghc->hva, len);
2013         if (r)
2014                 return -EFAULT;
2016         return 0;
2018 EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
2020 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
2022         const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
2024         return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
2026 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
2028 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
2030         gfn_t gfn = gpa >> PAGE_SHIFT;
2031         int seg;
2032         int offset = offset_in_page(gpa);
2033         int ret;
2035         while ((seg = next_segment(len, offset)) != 0) {
2036                 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
2037                 if (ret < 0)
2038                         return ret;
2039                 offset = 0;
2040                 len -= seg;
2041                 ++gfn;
2042         }
2043         return 0;
2045 EXPORT_SYMBOL_GPL(kvm_clear_guest);
2047 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot,
2048                                     gfn_t gfn)
2050         if (memslot && memslot->dirty_bitmap) {
2051                 unsigned long rel_gfn = gfn - memslot->base_gfn;
2053                 set_bit_le(rel_gfn, memslot->dirty_bitmap);
2054         }
2057 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
2059         struct kvm_memory_slot *memslot;
2061         memslot = gfn_to_memslot(kvm, gfn);
2062         mark_page_dirty_in_slot(memslot, gfn);
2064 EXPORT_SYMBOL_GPL(mark_page_dirty);
2066 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
2068         struct kvm_memory_slot *memslot;
2070         memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
2071         mark_page_dirty_in_slot(memslot, gfn);
2073 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
2075 void kvm_sigset_activate(struct kvm_vcpu *vcpu)
2077         if (!vcpu->sigset_active)
2078                 return;
2080         /*
2081          * This does a lockless modification of ->real_blocked, which is fine
2082          * because, only current can change ->real_blocked and all readers of
2083          * ->real_blocked don't care as long ->real_blocked is always a subset
2084          * of ->blocked.
2085          */
2086         sigprocmask(SIG_SETMASK, &vcpu->sigset, &current->real_blocked);
2089 void kvm_sigset_deactivate(struct kvm_vcpu *vcpu)
2091         if (!vcpu->sigset_active)
2092                 return;
2094         sigprocmask(SIG_SETMASK, &current->real_blocked, NULL);
2095         sigemptyset(&current->real_blocked);
2098 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
2100         unsigned int old, val, grow;
2102         old = val = vcpu->halt_poll_ns;
2103         grow = READ_ONCE(halt_poll_ns_grow);
2104         /* 10us base */
2105         if (val == 0 && grow)
2106                 val = 10000;
2107         else
2108                 val *= grow;
2110         if (val > halt_poll_ns)
2111                 val = halt_poll_ns;
2113         vcpu->halt_poll_ns = val;
2114         trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
2117 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
2119         unsigned int old, val, shrink;
2121         old = val = vcpu->halt_poll_ns;
2122         shrink = READ_ONCE(halt_poll_ns_shrink);
2123         if (shrink == 0)
2124                 val = 0;
2125         else
2126                 val /= shrink;
2128         vcpu->halt_poll_ns = val;
2129         trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
2132 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
2134         int ret = -EINTR;
2135         int idx = srcu_read_lock(&vcpu->kvm->srcu);
2137         if (kvm_arch_vcpu_runnable(vcpu)) {
2138                 kvm_make_request(KVM_REQ_UNHALT, vcpu);
2139                 goto out;
2140         }
2141         if (kvm_cpu_has_pending_timer(vcpu))
2142                 goto out;
2143         if (signal_pending(current))
2144                 goto out;
2146         ret = 0;
2147 out:
2148         srcu_read_unlock(&vcpu->kvm->srcu, idx);
2149         return ret;
2152 /*
2153  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
2154  */
2155 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
2157         ktime_t start, cur;
2158         DECLARE_SWAITQUEUE(wait);
2159         bool waited = false;
2160         u64 block_ns;
2162         start = cur = ktime_get();
2163         if (vcpu->halt_poll_ns) {
2164                 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
2166                 ++vcpu->stat.halt_attempted_poll;
2167                 do {
2168                         /*
2169                          * This sets KVM_REQ_UNHALT if an interrupt
2170                          * arrives.
2171                          */
2172                         if (kvm_vcpu_check_block(vcpu) < 0) {
2173                                 ++vcpu->stat.halt_successful_poll;
2174                                 if (!vcpu_valid_wakeup(vcpu))
2175                                         ++vcpu->stat.halt_poll_invalid;
2176                                 goto out;
2177                         }
2178                         cur = ktime_get();
2179                 } while (single_task_running() && ktime_before(cur, stop));
2180         }
2182         kvm_arch_vcpu_blocking(vcpu);
2184         for (;;) {
2185                 prepare_to_swait_exclusive(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
2187                 if (kvm_vcpu_check_block(vcpu) < 0)
2188                         break;
2190                 waited = true;
2191                 schedule();
2192         }
2194         finish_swait(&vcpu->wq, &wait);
2195         cur = ktime_get();
2197         kvm_arch_vcpu_unblocking(vcpu);
2198 out:
2199         block_ns = ktime_to_ns(cur) - ktime_to_ns(start);
2201         if (!vcpu_valid_wakeup(vcpu))
2202                 shrink_halt_poll_ns(vcpu);
2203         else if (halt_poll_ns) {
2204                 if (block_ns <= vcpu->halt_poll_ns)
2205                         ;
2206                 /* we had a long block, shrink polling */
2207                 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
2208                         shrink_halt_poll_ns(vcpu);
2209                 /* we had a short halt and our poll time is too small */
2210                 else if (vcpu->halt_poll_ns < halt_poll_ns &&
2211                         block_ns < halt_poll_ns)
2212                         grow_halt_poll_ns(vcpu);
2213         } else
2214                 vcpu->halt_poll_ns = 0;
2216         trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
2217         kvm_arch_vcpu_block_finish(vcpu);
2219 EXPORT_SYMBOL_GPL(kvm_vcpu_block);
2221 bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
2223         struct swait_queue_head *wqp;
2225         wqp = kvm_arch_vcpu_wq(vcpu);
2226         if (swq_has_sleeper(wqp)) {
2227                 swake_up_one(wqp);
2228                 ++vcpu->stat.halt_wakeup;
2229                 return true;
2230         }
2232         return false;
2234 EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);
2236 #ifndef CONFIG_S390
2237 /*
2238  * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
2239  */
2240 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
2242         int me;
2243         int cpu = vcpu->cpu;
2245         if (kvm_vcpu_wake_up(vcpu))
2246                 return;
2248         me = get_cpu();
2249         if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
2250                 if (kvm_arch_vcpu_should_kick(vcpu))
2251                         smp_send_reschedule(cpu);
2252         put_cpu();
2254 EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
2255 #endif /* !CONFIG_S390 */
2257 int kvm_vcpu_yield_to(struct kvm_vcpu *target)
2259         struct pid *pid;
2260         struct task_struct *task = NULL;
2261         int ret = 0;
2263         rcu_read_lock();
2264         pid = rcu_dereference(target->pid);
2265         if (pid)
2266                 task = get_pid_task(pid, PIDTYPE_PID);
2267         rcu_read_unlock();
2268         if (!task)
2269                 return ret;
2270         ret = yield_to(task, 1);
2271         put_task_struct(task);
2273         return ret;
2275 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
2277 /*
2278  * Helper that checks whether a VCPU is eligible for directed yield.
2279  * Most eligible candidate to yield is decided by following heuristics:
2280  *
2281  *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
2282  *  (preempted lock holder), indicated by @in_spin_loop.
2283  *  Set at the beiginning and cleared at the end of interception/PLE handler.
2284  *
2285  *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
2286  *  chance last time (mostly it has become eligible now since we have probably
2287  *  yielded to lockholder in last iteration. This is done by toggling
2288  *  @dy_eligible each time a VCPU checked for eligibility.)
2289  *
2290  *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
2291  *  to preempted lock-holder could result in wrong VCPU selection and CPU
2292  *  burning. Giving priority for a potential lock-holder increases lock
2293  *  progress.
2294  *
2295  *  Since algorithm is based on heuristics, accessing another VCPU data without
2296  *  locking does not harm. It may result in trying to yield to  same VCPU, fail
2297  *  and continue with next VCPU and so on.
2298  */
2299 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
2301 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
2302         bool eligible;
2304         eligible = !vcpu->spin_loop.in_spin_loop ||
2305                     vcpu->spin_loop.dy_eligible;
2307         if (vcpu->spin_loop.in_spin_loop)
2308                 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
2310         return eligible;
2311 #else
2312         return true;
2313 #endif
2316 void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode)
2318         struct kvm *kvm = me->kvm;
2319         struct kvm_vcpu *vcpu;
2320         int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
2321         int yielded = 0;
2322         int try = 3;
2323         int pass;
2324         int i;
2326         kvm_vcpu_set_in_spin_loop(me, true);
2327         /*
2328          * We boost the priority of a VCPU that is runnable but not
2329          * currently running, because it got preempted by something
2330          * else and called schedule in __vcpu_run.  Hopefully that
2331          * VCPU is holding the lock that we need and will release it.
2332          * We approximate round-robin by starting at the last boosted VCPU.
2333          */
2334         for (pass = 0; pass < 2 && !yielded && try; pass++) {
2335                 kvm_for_each_vcpu(i, vcpu, kvm) {
2336                         if (!pass && i <= last_boosted_vcpu) {
2337                                 i = last_boosted_vcpu;
2338                                 continue;
2339                         } else if (pass && i > last_boosted_vcpu)
2340                                 break;
2341                         if (!READ_ONCE(vcpu->preempted))
2342                                 continue;
2343                         if (vcpu == me)
2344                                 continue;
2345                         if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu))
2346                                 continue;
2347                         if (yield_to_kernel_mode && !kvm_arch_vcpu_in_kernel(vcpu))
2348                                 continue;
2349                         if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
2350                                 continue;
2352                         yielded = kvm_vcpu_yield_to(vcpu);
2353                         if (yielded > 0) {
2354                                 kvm->last_boosted_vcpu = i;
2355                                 break;
2356                         } else if (yielded < 0) {
2357                                 try--;
2358                                 if (!try)
2359                                         break;
2360                         }
2361                 }
2362         }
2363         kvm_vcpu_set_in_spin_loop(me, false);
2365         /* Ensure vcpu is not eligible during next spinloop */
2366         kvm_vcpu_set_dy_eligible(me, false);
2368 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
2370 static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf)
2372         struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
2373         struct page *page;
2375         if (vmf->pgoff == 0)
2376                 page = virt_to_page(vcpu->run);
2377 #ifdef CONFIG_X86
2378         else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
2379                 page = virt_to_page(vcpu->arch.pio_data);
2380 #endif
2381 #ifdef CONFIG_KVM_MMIO
2382         else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
2383                 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
2384 #endif
2385         else
2386                 return kvm_arch_vcpu_fault(vcpu, vmf);
2387         get_page(page);
2388         vmf->page = page;
2389         return 0;
2392 static const struct vm_operations_struct kvm_vcpu_vm_ops = {
2393         .fault = kvm_vcpu_fault,
2394 };
2396 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
2398         vma->vm_ops = &kvm_vcpu_vm_ops;
2399         return 0;
2402 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
2404         struct kvm_vcpu *vcpu = filp->private_data;
2406         debugfs_remove_recursive(vcpu->debugfs_dentry);
2407         kvm_put_kvm(vcpu->kvm);
2408         return 0;
2411 static struct file_operations kvm_vcpu_fops = {
2412         .release        = kvm_vcpu_release,
2413         .unlocked_ioctl = kvm_vcpu_ioctl,
2414         .mmap           = kvm_vcpu_mmap,
2415         .llseek         = noop_llseek,
2416         KVM_COMPAT(kvm_vcpu_compat_ioctl),
2417 };
2419 /*
2420  * Allocates an inode for the vcpu.
2421  */
2422 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
2424         char name[8 + 1 + ITOA_MAX_LEN + 1];
2426         snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id);
2427         return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
2430 static int kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
2432         char dir_name[ITOA_MAX_LEN * 2];
2433         int ret;
2435         if (!kvm_arch_has_vcpu_debugfs())
2436                 return 0;
2438         if (!debugfs_initialized())
2439                 return 0;
2441         snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id);
2442         vcpu->debugfs_dentry = debugfs_create_dir(dir_name,
2443                                                                 vcpu->kvm->debugfs_dentry);
2444         if (!vcpu->debugfs_dentry)
2445                 return -ENOMEM;
2447         ret = kvm_arch_create_vcpu_debugfs(vcpu);
2448         if (ret < 0) {
2449                 debugfs_remove_recursive(vcpu->debugfs_dentry);
2450                 return ret;
2451         }
2453         return 0;
2456 /*
2457  * Creates some virtual cpus.  Good luck creating more than one.
2458  */
2459 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
2461         int r;
2462         struct kvm_vcpu *vcpu;
2464         if (id >= KVM_MAX_VCPU_ID)
2465                 return -EINVAL;
2467         mutex_lock(&kvm->lock);
2468         if (kvm->created_vcpus == KVM_MAX_VCPUS) {
2469                 mutex_unlock(&kvm->lock);
2470                 return -EINVAL;
2471         }
2473         kvm->created_vcpus++;
2474         mutex_unlock(&kvm->lock);
2476         vcpu = kvm_arch_vcpu_create(kvm, id);
2477         if (IS_ERR(vcpu)) {
2478                 r = PTR_ERR(vcpu);
2479                 goto vcpu_decrement;
2480         }
2482         preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
2484         r = kvm_arch_vcpu_setup(vcpu);
2485         if (r)
2486                 goto vcpu_destroy;
2488         r = kvm_create_vcpu_debugfs(vcpu);
2489         if (r)
2490                 goto vcpu_destroy;
2492         mutex_lock(&kvm->lock);
2493         if (kvm_get_vcpu_by_id(kvm, id)) {
2494                 r = -EEXIST;
2495                 goto unlock_vcpu_destroy;
2496         }
2498         BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
2500         /* Now it's all set up, let userspace reach it */
2501         kvm_get_kvm(kvm);
2502         r = create_vcpu_fd(vcpu);
2503         if (r < 0) {
2504                 kvm_put_kvm(kvm);
2505                 goto unlock_vcpu_destroy;
2506         }
2508         kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
2510         /*
2511          * Pairs with smp_rmb() in kvm_get_vcpu.  Write kvm->vcpus
2512          * before kvm->online_vcpu's incremented value.
2513          */
2514         smp_wmb();
2515         atomic_inc(&kvm->online_vcpus);
2517         mutex_unlock(&kvm->lock);
2518         kvm_arch_vcpu_postcreate(vcpu);
2519         return r;
2521 unlock_vcpu_destroy:
2522         mutex_unlock(&kvm->lock);
2523         debugfs_remove_recursive(vcpu->debugfs_dentry);
2524 vcpu_destroy:
2525         kvm_arch_vcpu_destroy(vcpu);
2526 vcpu_decrement:
2527         mutex_lock(&kvm->lock);
2528         kvm->created_vcpus--;
2529         mutex_unlock(&kvm->lock);
2530         return r;
2533 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
2535         if (sigset) {
2536                 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
2537                 vcpu->sigset_active = 1;
2538                 vcpu->sigset = *sigset;
2539         } else
2540                 vcpu->sigset_active = 0;
2541         return 0;
2544 static long kvm_vcpu_ioctl(struct file *filp,
2545                            unsigned int ioctl, unsigned long arg)
2547         struct kvm_vcpu *vcpu = filp->private_data;
2548         void __user *argp = (void __user *)arg;
2549         int r;
2550         struct kvm_fpu *fpu = NULL;
2551         struct kvm_sregs *kvm_sregs = NULL;
2553         if (vcpu->kvm->mm != current->mm)
2554                 return -EIO;
2556         if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
2557                 return -EINVAL;
2559         /*
2560          * Some architectures have vcpu ioctls that are asynchronous to vcpu
2561          * execution; mutex_lock() would break them.
2562          */
2563         r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg);
2564         if (r != -ENOIOCTLCMD)
2565                 return r;
2567         if (mutex_lock_killable(&vcpu->mutex))
2568                 return -EINTR;
2569         switch (ioctl) {
2570         case KVM_RUN: {
2571                 struct pid *oldpid;
2572                 r = -EINVAL;
2573                 if (arg)
2574                         goto out;
2575                 oldpid = rcu_access_pointer(vcpu->pid);
2576                 if (unlikely(oldpid != task_pid(current))) {
2577                         /* The thread running this VCPU changed. */
2578                         struct pid *newpid;
2580                         r = kvm_arch_vcpu_run_pid_change(vcpu);
2581                         if (r)
2582                                 break;
2584                         newpid = get_task_pid(current, PIDTYPE_PID);
2585                         rcu_assign_pointer(vcpu->pid, newpid);
2586                         if (oldpid)
2587                                 synchronize_rcu();
2588                         put_pid(oldpid);
2589                 }
2590                 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
2591                 trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
2592                 break;
2593         }
2594         case KVM_GET_REGS: {
2595                 struct kvm_regs *kvm_regs;
2597                 r = -ENOMEM;
2598                 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
2599                 if (!kvm_regs)
2600                         goto out;
2601                 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
2602                 if (r)
2603                         goto out_free1;
2604                 r = -EFAULT;
2605                 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
2606                         goto out_free1;
2607                 r = 0;
2608 out_free1:
2609                 kfree(kvm_regs);
2610                 break;
2611         }
2612         case KVM_SET_REGS: {
2613                 struct kvm_regs *kvm_regs;
2615                 r = -ENOMEM;
2616                 kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
2617                 if (IS_ERR(kvm_regs)) {
2618                         r = PTR_ERR(kvm_regs);
2619                         goto out;
2620                 }
2621                 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
2622                 kfree(kvm_regs);
2623                 break;
2624         }
2625         case KVM_GET_SREGS: {
2626                 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
2627                 r = -ENOMEM;
2628                 if (!kvm_sregs)
2629                         goto out;
2630                 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
2631                 if (r)
2632                         goto out;
2633                 r = -EFAULT;
2634                 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
2635                         goto out;
2636                 r = 0;
2637                 break;
2638         }
2639         case KVM_SET_SREGS: {
2640                 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
2641                 if (IS_ERR(kvm_sregs)) {
2642                         r = PTR_ERR(kvm_sregs);
2643                         kvm_sregs = NULL;
2644                         goto out;
2645                 }
2646                 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
2647                 break;
2648         }
2649         case KVM_GET_MP_STATE: {
2650                 struct kvm_mp_state mp_state;
2652                 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
2653                 if (r)
2654                         goto out;
2655                 r = -EFAULT;
2656                 if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
2657                         goto out;
2658                 r = 0;
2659                 break;
2660         }
2661         case KVM_SET_MP_STATE: {
2662                 struct kvm_mp_state mp_state;
2664                 r = -EFAULT;
2665                 if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
2666                         goto out;
2667                 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
2668                 break;
2669         }
2670         case KVM_TRANSLATE: {
2671                 struct kvm_translation tr;
2673                 r = -EFAULT;
2674                 if (copy_from_user(&tr, argp, sizeof(tr)))
2675                         goto out;
2676                 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
2677                 if (r)
2678                         goto out;
2679                 r = -EFAULT;
2680                 if (copy_to_user(argp, &tr, sizeof(tr)))
2681                         goto out;
2682                 r = 0;
2683                 break;
2684         }
2685         case KVM_SET_GUEST_DEBUG: {
2686                 struct kvm_guest_debug dbg;
2688                 r = -EFAULT;
2689                 if (copy_from_user(&dbg, argp, sizeof(dbg)))
2690                         goto out;
2691                 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
2692                 break;
2693         }
2694         case KVM_SET_SIGNAL_MASK: {
2695                 struct kvm_signal_mask __user *sigmask_arg = argp;
2696                 struct kvm_signal_mask kvm_sigmask;
2697                 sigset_t sigset, *p;
2699                 p = NULL;
2700                 if (argp) {
2701                         r = -EFAULT;
2702                         if (copy_from_user(&kvm_sigmask, argp,
2703                                            sizeof(kvm_sigmask)))
2704                                 goto out;
2705                         r = -EINVAL;
2706                         if (kvm_sigmask.len != sizeof(sigset))
2707                                 goto out;
2708                         r = -EFAULT;
2709                         if (copy_from_user(&sigset, sigmask_arg->sigset,
2710                                            sizeof(sigset)))
2711                                 goto out;
2712                         p = &sigset;
2713                 }
2714                 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
2715                 break;
2716         }
2717         case KVM_GET_FPU: {
2718                 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
2719                 r = -ENOMEM;
2720                 if (!fpu)
2721                         goto out;
2722                 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
2723                 if (r)
2724                         goto out;
2725                 r = -EFAULT;
2726                 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
2727                         goto out;
2728                 r = 0;
2729                 break;
2730         }
2731         case KVM_SET_FPU: {
2732                 fpu = memdup_user(argp, sizeof(*fpu));
2733                 if (IS_ERR(fpu)) {
2734                         r = PTR_ERR(fpu);
2735                         fpu = NULL;
2736                         goto out;
2737                 }
2738                 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
2739                 break;
2740         }
2741         default:
2742                 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
2743         }
2744 out:
2745         mutex_unlock(&vcpu->mutex);
2746         kfree(fpu);
2747         kfree(kvm_sregs);
2748         return r;
2751 #ifdef CONFIG_KVM_COMPAT
2752 static long kvm_vcpu_compat_ioctl(struct file *filp,
2753                                   unsigned int ioctl, unsigned long arg)
2755         struct kvm_vcpu *vcpu = filp->private_data;
2756         void __user *argp = compat_ptr(arg);
2757         int r;
2759         if (vcpu->kvm->mm != current->mm)
2760                 return -EIO;
2762         switch (ioctl) {
2763         case KVM_SET_SIGNAL_MASK: {
2764                 struct kvm_signal_mask __user *sigmask_arg = argp;
2765                 struct kvm_signal_mask kvm_sigmask;
2766                 sigset_t sigset;
2768                 if (argp) {
2769                         r = -EFAULT;
2770                         if (copy_from_user(&kvm_sigmask, argp,
2771                                            sizeof(kvm_sigmask)))
2772                                 goto out;
2773                         r = -EINVAL;
2774                         if (kvm_sigmask.len != sizeof(compat_sigset_t))
2775                                 goto out;
2776                         r = -EFAULT;
2777                         if (get_compat_sigset(&sigset, (void *)sigmask_arg->sigset))
2778                                 goto out;
2779                         r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
2780                 } else
2781                         r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2782                 break;
2783         }
2784         default:
2785                 r = kvm_vcpu_ioctl(filp, ioctl, arg);
2786         }
2788 out:
2789         return r;
2791 #endif
2793 static int kvm_device_ioctl_attr(struct kvm_device *dev,
2794                                  int (*accessor)(struct kvm_device *dev,
2795                                                  struct kvm_device_attr *attr),
2796                                  unsigned long arg)
2798         struct kvm_device_attr attr;
2800         if (!accessor)
2801                 return -EPERM;
2803         if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2804                 return -EFAULT;
2806         return accessor(dev, &attr);
2809 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
2810                              unsigned long arg)
2812         struct kvm_device *dev = filp->private_data;
2814         switch (ioctl) {
2815         case KVM_SET_DEVICE_ATTR:
2816                 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
2817         case KVM_GET_DEVICE_ATTR:
2818                 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
2819         case KVM_HAS_DEVICE_ATTR:
2820                 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
2821         default:
2822                 if (dev->ops->ioctl)
2823                         return dev->ops->ioctl(dev, ioctl, arg);
2825                 return -ENOTTY;
2826         }
2829 static int kvm_device_release(struct inode *inode, struct file *filp)
2831         struct kvm_device *dev = filp->private_data;
2832         struct kvm *kvm = dev->kvm;
2834         kvm_put_kvm(kvm);
2835         return 0;
2838 static const struct file_operations kvm_device_fops = {
2839         .unlocked_ioctl = kvm_device_ioctl,
2840         .release = kvm_device_release,
2841         KVM_COMPAT(kvm_device_ioctl),
2842 };
2844 struct kvm_device *kvm_device_from_filp(struct file *filp)
2846         if (filp->f_op != &kvm_device_fops)
2847                 return NULL;
2849         return filp->private_data;
2852 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
2853 #ifdef CONFIG_KVM_MPIC
2854         [KVM_DEV_TYPE_FSL_MPIC_20]      = &kvm_mpic_ops,
2855         [KVM_DEV_TYPE_FSL_MPIC_42]      = &kvm_mpic_ops,
2856 #endif
2857 };
2859 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type)
2861         if (type >= ARRAY_SIZE(kvm_device_ops_table))
2862                 return -ENOSPC;
2864         if (kvm_device_ops_table[type] != NULL)
2865                 return -EEXIST;
2867         kvm_device_ops_table[type] = ops;
2868         return 0;
2871 void kvm_unregister_device_ops(u32 type)
2873         if (kvm_device_ops_table[type] != NULL)
2874                 kvm_device_ops_table[type] = NULL;
2877 static int kvm_ioctl_create_device(struct kvm *kvm,
2878                                    struct kvm_create_device *cd)
2880         struct kvm_device_ops *ops = NULL;
2881         struct kvm_device *dev;
2882         bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
2883         int ret;
2885         if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
2886                 return -ENODEV;
2888         ops = kvm_device_ops_table[cd->type];
2889         if (ops == NULL)
2890                 return -ENODEV;
2892         if (test)
2893                 return 0;
2895         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2896         if (!dev)
2897                 return -ENOMEM;
2899         dev->ops = ops;
2900         dev->kvm = kvm;
2902         mutex_lock(&kvm->lock);
2903         ret = ops->create(dev, cd->type);
2904         if (ret < 0) {
2905                 mutex_unlock(&kvm->lock);
2906                 kfree(dev);
2907                 return ret;
2908         }
2909         list_add(&dev->vm_node, &kvm->devices);
2910         mutex_unlock(&kvm->lock);
2912         if (ops->init)
2913                 ops->init(dev);
2915         ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
2916         if (ret < 0) {
2917                 mutex_lock(&kvm->lock);
2918                 list_del(&dev->vm_node);
2919                 mutex_unlock(&kvm->lock);
2920                 ops->destroy(dev);
2921                 return ret;
2922         }
2924         kvm_get_kvm(kvm);
2925         cd->fd = ret;
2926         return 0;
2929 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
2931         switch (arg) {
2932         case KVM_CAP_USER_MEMORY:
2933         case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2934         case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2935         case KVM_CAP_INTERNAL_ERROR_DATA:
2936 #ifdef CONFIG_HAVE_KVM_MSI
2937         case KVM_CAP_SIGNAL_MSI:
2938 #endif
2939 #ifdef CONFIG_HAVE_KVM_IRQFD
2940         case KVM_CAP_IRQFD:
2941         case KVM_CAP_IRQFD_RESAMPLE:
2942 #endif
2943         case KVM_CAP_IOEVENTFD_ANY_LENGTH:
2944         case KVM_CAP_CHECK_EXTENSION_VM:
2945                 return 1;
2946 #ifdef CONFIG_KVM_MMIO
2947         case KVM_CAP_COALESCED_MMIO:
2948                 return KVM_COALESCED_MMIO_PAGE_OFFSET;
2949 #endif
2950 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
2951         case KVM_CAP_IRQ_ROUTING:
2952                 return KVM_MAX_IRQ_ROUTES;
2953 #endif
2954 #if KVM_ADDRESS_SPACE_NUM > 1
2955         case KVM_CAP_MULTI_ADDRESS_SPACE:
2956                 return KVM_ADDRESS_SPACE_NUM;
2957 #endif
2958         case KVM_CAP_MAX_VCPU_ID:
2959                 return KVM_MAX_VCPU_ID;
2960         default:
2961                 break;
2962         }
2963         return kvm_vm_ioctl_check_extension(kvm, arg);
2966 static long kvm_vm_ioctl(struct file *filp,
2967                            unsigned int ioctl, unsigned long arg)
2969         struct kvm *kvm = filp->private_data;
2970         void __user *argp = (void __user *)arg;
2971         int r;
2973         if (kvm->mm != current->mm)
2974                 return -EIO;
2975         switch (ioctl) {
2976         case KVM_CREATE_VCPU:
2977                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2978                 break;
2979         case KVM_SET_USER_MEMORY_REGION: {
2980                 struct kvm_userspace_memory_region kvm_userspace_mem;
2982                 r = -EFAULT;
2983                 if (copy_from_user(&kvm_userspace_mem, argp,
2984                                                 sizeof(kvm_userspace_mem)))
2985                         goto out;
2987                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
2988                 break;
2989         }
2990         case KVM_GET_DIRTY_LOG: {
2991                 struct kvm_dirty_log log;
2993                 r = -EFAULT;
2994                 if (copy_from_user(&log, argp, sizeof(log)))
2995                         goto out;
2996                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2997                 break;
2998         }
2999 #ifdef CONFIG_KVM_MMIO
3000         case KVM_REGISTER_COALESCED_MMIO: {
3001                 struct kvm_coalesced_mmio_zone zone;
3003                 r = -EFAULT;
3004                 if (copy_from_user(&zone, argp, sizeof(zone)))
3005                         goto out;
3006                 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
3007                 break;
3008         }
3009         case KVM_UNREGISTER_COALESCED_MMIO: {
3010                 struct kvm_coalesced_mmio_zone zone;
3012                 r = -EFAULT;
3013                 if (copy_from_user(&zone, argp, sizeof(zone)))
3014                         goto out;
3015                 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
3016                 break;
3017         }
3018 #endif
3019         case KVM_IRQFD: {
3020                 struct kvm_irqfd data;
3022                 r = -EFAULT;
3023                 if (copy_from_user(&data, argp, sizeof(data)))
3024                         goto out;
3025                 r = kvm_irqfd(kvm, &data);
3026                 break;
3027         }
3028         case KVM_IOEVENTFD: {
3029                 struct kvm_ioeventfd data;
3031                 r = -EFAULT;
3032                 if (copy_from_user(&data, argp, sizeof(data)))
3033                         goto out;
3034                 r = kvm_ioeventfd(kvm, &data);
3035                 break;
3036         }
3037 #ifdef CONFIG_HAVE_KVM_MSI
3038         case KVM_SIGNAL_MSI: {
3039                 struct kvm_msi msi;
3041                 r = -EFAULT;
3042                 if (copy_from_user(&msi, argp, sizeof(msi)))
3043                         goto out;
3044                 r = kvm_send_userspace_msi(kvm, &msi);
3045                 break;
3046         }
3047 #endif
3048 #ifdef __KVM_HAVE_IRQ_LINE
3049         case KVM_IRQ_LINE_STATUS:
3050         case KVM_IRQ_LINE: {
3051                 struct kvm_irq_level irq_event;
3053                 r = -EFAULT;
3054                 if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
3055                         goto out;
3057                 r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
3058                                         ioctl == KVM_IRQ_LINE_STATUS);
3059                 if (r)
3060                         goto out;
3062                 r = -EFAULT;
3063                 if (ioctl == KVM_IRQ_LINE_STATUS) {
3064                         if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
3065                                 goto out;
3066                 }
3068                 r = 0;
3069                 break;
3070         }
3071 #endif
3072 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
3073         case KVM_SET_GSI_ROUTING: {
3074                 struct kvm_irq_routing routing;
3075                 struct kvm_irq_routing __user *urouting;
3076                 struct kvm_irq_routing_entry *entries = NULL;
3078                 r = -EFAULT;
3079                 if (copy_from_user(&routing, argp, sizeof(routing)))
3080                         goto out;
3081                 r = -EINVAL;
3082                 if (!kvm_arch_can_set_irq_routing(kvm))
3083                         goto out;
3084                 if (routing.nr > KVM_MAX_IRQ_ROUTES)
3085                         goto out;
3086                 if (routing.flags)
3087                         goto out;
3088                 if (routing.nr) {
3089                         r = -ENOMEM;
3090                         entries = vmalloc(array_size(sizeof(*entries),
3091                                                      routing.nr));
3092                         if (!entries)
3093                                 goto out;
3094                         r = -EFAULT;
3095                         urouting = argp;
3096                         if (copy_from_user(entries, urouting->entries,
3097                                            routing.nr * sizeof(*entries)))
3098                                 goto out_free_irq_routing;
3099                 }
3100                 r = kvm_set_irq_routing(kvm, entries, routing.nr,
3101                                         routing.flags);
3102 out_free_irq_routing:
3103                 vfree(entries);
3104                 break;
3105         }
3106 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
3107         case KVM_CREATE_DEVICE: {
3108                 struct kvm_create_device cd;
3110                 r = -EFAULT;
3111                 if (copy_from_user(&cd, argp, sizeof(cd)))
3112                         goto out;
3114                 r = kvm_ioctl_create_device(kvm, &cd);
3115                 if (r)
3116                         goto out;
3118                 r = -EFAULT;
3119                 if (copy_to_user(argp, &cd, sizeof(cd)))
3120                         goto out;
3122                 r = 0;
3123                 break;
3124         }
3125         case KVM_CHECK_EXTENSION:
3126                 r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
3127                 break;
3128         default:
3129                 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
3130         }
3131 out:
3132         return r;
3135 #ifdef CONFIG_KVM_COMPAT
3136 struct compat_kvm_dirty_log {
3137         __u32 slot;
3138         __u32 padding1;
3139         union {
3140                 compat_uptr_t dirty_bitmap; /* one bit per page */
3141                 __u64 padding2;
3142         };
3143 };
3145 static long kvm_vm_compat_ioctl(struct file *filp,
3146                            unsigned int ioctl, unsigned long arg)
3148         struct kvm *kvm = filp->private_data;
3149         int r;
3151         if (kvm->mm != current->mm)
3152                 return -EIO;
3153         switch (ioctl) {
3154         case KVM_GET_DIRTY_LOG: {
3155                 struct compat_kvm_dirty_log compat_log;
3156                 struct kvm_dirty_log log;
3158                 if (copy_from_user(&compat_log, (void __user *)arg,
3159                                    sizeof(compat_log)))
3160                         return -EFAULT;
3161                 log.slot         = compat_log.slot;
3162                 log.padding1     = compat_log.padding1;
3163                 log.padding2     = compat_log.padding2;
3164                 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
3166                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
3167                 break;
3168         }
3169         default:
3170                 r = kvm_vm_ioctl(filp, ioctl, arg);
3171         }
3172         return r;
3174 #endif
3176 static struct file_operations kvm_vm_fops = {
3177         .release        = kvm_vm_release,
3178         .unlocked_ioctl = kvm_vm_ioctl,
3179         .llseek         = noop_llseek,
3180         KVM_COMPAT(kvm_vm_compat_ioctl),
3181 };
3183 static int kvm_dev_ioctl_create_vm(unsigned long type)
3185         int r;
3186         struct kvm *kvm;
3187         struct file *file;
3189         kvm = kvm_create_vm(type);
3190         if (IS_ERR(kvm))
3191                 return PTR_ERR(kvm);
3192 #ifdef CONFIG_KVM_MMIO
3193         r = kvm_coalesced_mmio_init(kvm);
3194         if (r < 0)
3195                 goto put_kvm;
3196 #endif
3197         r = get_unused_fd_flags(O_CLOEXEC);
3198         if (r < 0)
3199                 goto put_kvm;
3201         file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
3202         if (IS_ERR(file)) {
3203                 put_unused_fd(r);
3204                 r = PTR_ERR(file);
3205                 goto put_kvm;
3206         }
3208         /*
3209          * Don't call kvm_put_kvm anymore at this point; file->f_op is
3210          * already set, with ->release() being kvm_vm_release().  In error
3211          * cases it will be called by the final fput(file) and will take
3212          * care of doing kvm_put_kvm(kvm).
3213          */
3214         if (kvm_create_vm_debugfs(kvm, r) < 0) {
3215                 put_unused_fd(r);
3216                 fput(file);
3217                 return -ENOMEM;
3218         }
3219         kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
3221         fd_install(r, file);
3222         return r;
3224 put_kvm:
3225         kvm_put_kvm(kvm);
3226         return r;
3229 static long kvm_dev_ioctl(struct file *filp,
3230                           unsigned int ioctl, unsigned long arg)
3232         long r = -EINVAL;
3234         switch (ioctl) {
3235         case KVM_GET_API_VERSION:
3236                 if (arg)
3237                         goto out;
3238                 r = KVM_API_VERSION;
3239                 break;
3240         case KVM_CREATE_VM:
3241                 r = kvm_dev_ioctl_create_vm(arg);
3242                 break;
3243         case KVM_CHECK_EXTENSION:
3244                 r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
3245                 break;
3246         case KVM_GET_VCPU_MMAP_SIZE:
3247                 if (arg)
3248                         goto out;
3249                 r = PAGE_SIZE;     /* struct kvm_run */
3250 #ifdef CONFIG_X86
3251                 r += PAGE_SIZE;    /* pio data page */
3252 #endif
3253 #ifdef CONFIG_KVM_MMIO
3254                 r += PAGE_SIZE;    /* coalesced mmio ring page */
3255 #endif
3256                 break;
3257         case KVM_TRACE_ENABLE:
3258         case KVM_TRACE_PAUSE:
3259         case KVM_TRACE_DISABLE:
3260                 r = -EOPNOTSUPP;
3261                 break;
3262         default:
3263                 return kvm_arch_dev_ioctl(filp, ioctl, arg);
3264         }
3265 out:
3266         return r;
3269 static struct file_operations kvm_chardev_ops = {
3270         .unlocked_ioctl = kvm_dev_ioctl,
3271         .llseek         = noop_llseek,
3272         KVM_COMPAT(kvm_dev_ioctl),
3273 };
3275 static struct miscdevice kvm_dev = {
3276         KVM_MINOR,
3277         "kvm",
3278         &kvm_chardev_ops,
3279 };
3281 static void hardware_enable_nolock(void *junk)
3283         int cpu = raw_smp_processor_id();
3284         int r;
3286         if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
3287                 return;
3289         cpumask_set_cpu(cpu, cpus_hardware_enabled);
3291         r = kvm_arch_hardware_enable();
3293         if (r) {
3294                 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3295                 atomic_inc(&hardware_enable_failed);
3296                 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
3297         }
3300 static int kvm_starting_cpu(unsigned int cpu)
3302         raw_spin_lock(&kvm_count_lock);
3303         if (kvm_usage_count)
3304                 hardware_enable_nolock(NULL);
3305         raw_spin_unlock(&kvm_count_lock);
3306         return 0;
3309 static void hardware_disable_nolock(void *junk)
3311         int cpu = raw_smp_processor_id();
3313         if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
3314                 return;
3315         cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3316         kvm_arch_hardware_disable();
3319 static int kvm_dying_cpu(unsigned int cpu)
3321         raw_spin_lock(&kvm_count_lock);
3322         if (kvm_usage_count)
3323                 hardware_disable_nolock(NULL);
3324         raw_spin_unlock(&kvm_count_lock);
3325         return 0;
3328 static void hardware_disable_all_nolock(void)
3330         BUG_ON(!kvm_usage_count);
3332         kvm_usage_count--;
3333         if (!kvm_usage_count)
3334                 on_each_cpu(hardware_disable_nolock, NULL, 1);
3337 static void hardware_disable_all(void)
3339         raw_spin_lock(&kvm_count_lock);
3340         hardware_disable_all_nolock();
3341         raw_spin_unlock(&kvm_count_lock);
3344 static int hardware_enable_all(void)
3346         int r = 0;
3348         raw_spin_lock(&kvm_count_lock);
3350         kvm_usage_count++;
3351         if (kvm_usage_count == 1) {
3352                 atomic_set(&hardware_enable_failed, 0);
3353                 on_each_cpu(hardware_enable_nolock, NULL, 1);
3355                 if (atomic_read(&hardware_enable_failed)) {
3356                         hardware_disable_all_nolock();
3357                         r = -EBUSY;
3358                 }
3359         }
3361         raw_spin_unlock(&kvm_count_lock);
3363         return r;
3366 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
3367                       void *v)
3369         /*
3370          * Some (well, at least mine) BIOSes hang on reboot if
3371          * in vmx root mode.
3372          *
3373          * And Intel TXT required VMX off for all cpu when system shutdown.
3374          */
3375         pr_info("kvm: exiting hardware virtualization\n");
3376         kvm_rebooting = true;
3377         on_each_cpu(hardware_disable_nolock, NULL, 1);
3378         return NOTIFY_OK;
3381 static struct notifier_block kvm_reboot_notifier = {
3382         .notifier_call = kvm_reboot,
3383         .priority = 0,
3384 };
3386 static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
3388         int i;
3390         for (i = 0; i < bus->dev_count; i++) {
3391                 struct kvm_io_device *pos = bus->range[i].dev;
3393                 kvm_iodevice_destructor(pos);
3394         }
3395         kfree(bus);
3398 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
3399                                  const struct kvm_io_range *r2)
3401         gpa_t addr1 = r1->addr;
3402         gpa_t addr2 = r2->addr;
3404         if (addr1 < addr2)
3405                 return -1;
3407         /* If r2->len == 0, match the exact address.  If r2->len != 0,
3408          * accept any overlapping write.  Any order is acceptable for
3409          * overlapping ranges, because kvm_io_bus_get_first_dev ensures
3410          * we process all of them.
3411          */
3412         if (r2->len) {
3413                 addr1 += r1->len;
3414                 addr2 += r2->len;
3415         }
3417         if (addr1 > addr2)
3418                 return 1;
3420         return 0;
3423 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
3425         return kvm_io_bus_cmp(p1, p2);
3428 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
3429                              gpa_t addr, int len)
3431         struct kvm_io_range *range, key;
3432         int off;
3434         key = (struct kvm_io_range) {
3435                 .addr = addr,
3436                 .len = len,
3437         };
3439         range = bsearch(&key, bus->range, bus->dev_count,
3440                         sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
3441         if (range == NULL)
3442                 return -ENOENT;
3444         off = range - bus->range;
3446         while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
3447                 off--;
3449         return off;
3452 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3453                               struct kvm_io_range *range, const void *val)
3455         int idx;
3457         idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3458         if (idx < 0)
3459                 return -EOPNOTSUPP;
3461         while (idx < bus->dev_count &&
3462                 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3463                 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
3464                                         range->len, val))
3465                         return idx;
3466                 idx++;
3467         }
3469         return -EOPNOTSUPP;
3472 /* kvm_io_bus_write - called under kvm->slots_lock */
3473 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3474                      int len, const void *val)
3476         struct kvm_io_bus *bus;
3477         struct kvm_io_range range;
3478         int r;
3480         range = (struct kvm_io_range) {
3481                 .addr = addr,
3482                 .len = len,
3483         };
3485         bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3486         if (!bus)
3487                 return -ENOMEM;
3488         r = __kvm_io_bus_write(vcpu, bus, &range, val);
3489         return r < 0 ? r : 0;
3492 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */
3493 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
3494                             gpa_t addr, int len, const void *val, long cookie)
3496         struct kvm_io_bus *bus;
3497         struct kvm_io_range range;
3499         range = (struct kvm_io_range) {
3500                 .addr = addr,
3501                 .len = len,
3502         };
3504         bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3505         if (!bus)
3506                 return -ENOMEM;
3508         /* First try the device referenced by cookie. */
3509         if ((cookie >= 0) && (cookie < bus->dev_count) &&
3510             (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
3511                 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
3512                                         val))
3513                         return cookie;
3515         /*
3516          * cookie