/* kvm设置硬件特性函数 */
int kvm_arch_hardware_setup(void *opaque)
{
	struct kvm_x86_init_ops *ops = opaque;
	int r;

	rdmsrl_safe(MSR_EFER, &host_efer);

	if (boot_cpu_has(X86_FEATURE_XSAVES))
		rdmsrl(MSR_IA32_XSS, host_xss);
	/* 调用vmx或者svm的hardware_setup */
	r = ops->hardware_setup();
	if (r != 0)
		return r;

	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
	kvm_ops_static_call_update();

	if (ops->intel_pt_intr_in_guest && ops->intel_pt_intr_in_guest())
		kvm_guest_cbs.handle_intel_pt_intr = kvm_handle_intel_pt_intr;
	perf_register_guest_info_callbacks(&kvm_guest_cbs);

	/* 初始化msr list*/
	kvm_init_msr_list();
	return 0;
}

/* kvm初始化函数 */
int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
		  struct module *module)
{
	struct kvm_cpu_compat_check c;
	int r;
	int cpu;
    /* 初始化x86架构,检查bios cpu等特性 */
	r = kvm_arch_init(opaque);
	if (r)
		goto out_fail;

	/*
	 * kvm_arch_init makes sure there's at most one caller
	 * for architectures that support multiple implementations,
	 * like intel and amd on x86.
	 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
	 * conflicts in case kvm is already setup for another implementation.
	 */
	 
	 /* 初始化中断虚拟化 */
	r = kvm_irqfd_init();
	if (r)
		goto out_irqfd;

	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
		r = -ENOMEM;
		goto out_free_0;
	}
    /* 设置硬件相关 其中通过ops函数指针调用vmx的hardware_setup*/
	r = kvm_arch_hardware_setup(opaque);
	if (r < 0)
		goto out_free_1;

	c.ret = &r;
	c.opaque = opaque;
	/* 检查cpu兼容性 */
	for_each_online_cpu(cpu) {
		smp_call_function_single(cpu, check_processor_compat, &c, 1);
		if (r < 0)
			goto out_free_2;
	}

	r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "kvm/cpu:starting",
				      kvm_starting_cpu, kvm_dying_cpu);
	if (r)
		goto out_free_2;
	register_reboot_notifier(&kvm_reboot_notifier);

	...

	kvm_init_debug();
    /* 初始化vfio相关 供对直接 I/O 访问硬件设备的支持 */
	r = kvm_vfio_ops_init();
	WARN_ON(r);

	return 0;

out_unreg:
	kvm_async_pf_deinit();
out_free:
	kmem_cache_destroy(kvm_vcpu_cache);
out_free_3:
	unregister_reboot_notifier(&kvm_reboot_notifier);
	cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
out_free_2:
	kvm_arch_hardware_unsetup();
out_free_1:
	free_cpumask_var(cpus_hardware_enabled);
out_free_0:
	kvm_irqfd_exit();
out_irqfd:
	kvm_arch_exit();
out_fail:
	return r;
}
EXPORT_SYMBOL_GPL(kvm_init);

/*
vmx intel 设置硬件注册函数，由kvm kvm_arch_hardware_setup调用
*/
static __init int hardware_setup(void){
    ...
    vmx_setup_user_return_msrs();
	/* 设置vmcs配置文件 */
	if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0)
		return -EIO;
    /* 设置或更新当前支持的 CPU 特性*/
    vmx_set_cpu_caps();
    
	/* 为每个cpu设置vmcs */
	r = alloc_kvm_area();
	if (r)
		nested_vmx_hardware_unsetup();

	/* 设置 host的中断唤醒回调处理函数 */
	kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler);

	return r;
}

/* 关键数据结构vmcs */
struct vmcs_hdr {
	u32 revision_id:31;
	u32 shadow_vmcs:1;
};
/* 关键数据结构vmcs */
struct vmcs {
	struct vmcs_hdr hdr;
	u32 abort;
	char data[];
};


/* 为每个cpu设置vmcs */
static __init int alloc_kvm_area(void)
{
	int cpu;

	for_each_possible_cpu(cpu) {
		struct vmcs *vmcs;

		vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL);
		if (!vmcs) {
			free_kvm_area();
			return -ENOMEM;
		}
		if (static_branch_unlikely(&enable_evmcs))
			vmcs->hdr.revision_id = vmcs_config.revision_id;

		per_cpu(vmxarea, cpu) = vmcs;
	}
	return 0;
}

/* 中断唤醒回调处理函数 */
void pi_wakeup_handler(void)
{
	struct kvm_vcpu *vcpu;
	int cpu = smp_processor_id();

	raw_spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
	list_for_each_entry(vcpu, &per_cpu(blocked_vcpu_on_cpu, cpu),
			blocked_vcpu_list) {
		struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);

		if (pi_test_on(pi_desc) == 1)
            /* kick一下cpu */
			kvm_vcpu_kick(vcpu);
	}
	raw_spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
}

/* kick cpu 函数 */
void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
{
	int me, cpu;

	if (kvm_vcpu_wake_up(vcpu))
		return;

	/*
	 * Note, the vCPU could get migrated to a different pCPU at any point
	 * after kvm_arch_vcpu_should_kick(), which could result in sending an
	 * IPI to the previous pCPU.  But, that's ok because the purpose of the
	 * IPI is to force the vCPU to leave IN_GUEST_MODE, and migrating the
	 * vCPU also requires it to leave IN_GUEST_MODE.
	 */
	me = get_cpu();
	if (kvm_arch_vcpu_should_kick(vcpu)) {
		cpu = READ_ONCE(vcpu->cpu);
		if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
            /* 发送IPI，进行重新调度task*/
			smp_send_reschedule(cpu);
	}
	put_cpu();
}

static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
{
	int ret = __vmx_handle_exit(vcpu, exit_fastpath);

	if (to_vmx(vcpu)->exit_reason.bus_lock_detected) {
		if (ret > 0)
			vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK;

		vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK;
		return 0;
	}
	return ret;
}

static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
{
    ...
    /* 是WRITE_MSR 原因导致的vm_exit*/
	if (exit_reason.basic == EXIT_REASON_MSR_WRITE)
		return kvm_emulate_wrmsr(vcpu);
	...
}
/* 处理msr 写入*/
int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
{
	u32 ecx = kvm_rcx_read(vcpu);
	u64 data = kvm_read_edx_eax(vcpu);
	int r;
    /* 静态调用机制 kvm_x86_complete_emulated_msr 即 vmx_set_msr*/
	r = kvm_set_msr(vcpu, ecx, data);

	/* MSR write failed? See if we should ask user space */
	if (r && kvm_set_msr_user_space(vcpu, ecx, data, r))
		/* Bounce to user space */
		return 0;

	/* Signal all other negative errors to userspace */
	if (r < 0)
		return r;

	if (!r)
		trace_kvm_msr_write(ecx, data);
	else
		trace_kvm_msr_write_ex(ecx, data);
	
	return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
}

int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
{
	return kvm_set_msr_ignored_check(vcpu, index, data, false);
}

static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,
				     u32 index, u64 data, bool host_initiated)
{
	int ret = __kvm_set_msr(vcpu, index, data, host_initiated);

	if (ret == KVM_MSR_RET_INVALID)
        /* 是否忽略msr不可用返回*/
		if (kvm_msr_ignored_check(index, data, true))
			ret = 0;

	return ret;
}

static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
			 bool host_initiated)
{
    ...
    /* 静态调用机制是内核的一种优化手段，目的是减少动态函数指针调用的性能开销。在传统动态调用中，
    调用函数指针需要额外的间接跳转开销（例如通过寄存器保存目标地址）。静态调用通过在编译时或运行时
    将调用点的目标函数直接替换成最终实现函数，避免了这一开销 */
    
    /*最终调用vmx_set_msr */
    return static_call(kvm_x86_set_msr)(vcpu, &msr);
}

/*
 * Writes msr value into the appropriate "register".
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info){
    struct vcpu_vmx *vmx = to_vmx(vcpu);
	struct vmx_uret_msr *msr;
	int ret = 0;
	u32 msr_index = msr_info->index;
	u64 data = msr_info->data;
	u32 index;

	switch (msr_index) {
    
    ...
	case MSR_IA32_DEBUGCTLMSR:
		if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls &
						VM_EXIT_SAVE_DEBUG_CONTROLS)
			get_vmcs12(vcpu)->guest_ia32_debugctl = data;

		ret = kvm_set_msr_common(vcpu, msr_info);
		break;
	}
    ...
}

int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
	bool pr = false;
	u32 msr = msr_info->index;
	u64 data = msr_info->data;

	switch (msr) {
    ...
    case MSR_IA32_DEBUGCTLMSR:
		if (!data) {
			/* We support the non-activated case already */
			break;
		} else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
			/* Values other than LBR and BTF are vendor-specific,
			   thus reserved and should throw a #GP */
			return 1;
		} else if (report_ignored_msrs)
			vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
				    __func__, data);
		break;
    }
    ...
}