Hot-Plug Init

PCIe hot-plug是一种支持在不关机情况下从支持的插槽添加或删除设备的功能，PCIe架构定义了一些寄存器以支持原生热插拔。相关寄存器主要分布在Device Capabilities, Slot Capabilities, Slot Control, Slot Status和Slot Capabilities 2。Hot-Plug相关支持寄存器位置如下。

一个Downstream Port支持以下热插拔事件。

• Slot Events:
  • Attention Button Pressed
  • Power Fault Detected
  • MRL Sensor Changed
  • Presence Detect Changed
• Command Completed Events
• Data Link Layer State Changed Events

主要事件都可以通过寄存器控制使能，需要在初始化时配置启用。如果要启用INTx message，需配置如下：

• Command register内的Interrupt Disable位需要设为0
• Slot Control register内的Hot-Plug Interrupt Enable位需要设为1
• 至少使能一个hot-plug event控制寄存器
  如果要启用MSI/MSI-X，则需取消屏蔽关联向量。PME和Hot-Plug事件中断共享同一向量，由PCIE Capabilities的Interrupt Message Number域段控制。
  

Hot-Plug事件产生需要启用PME_En位。

Hot Add Flow

主要处理Presence Detect Changed, Command Completed events（需要在No Command Completed Support为0时，否则硬件支持无通知式自动完成）。
pciehp_handle_presence_or_link_change函数为Presence Detect Changed事件中断服务函数。

void pciehp_handle_presence_or_link_change(struct controller *ctrl, u32 events)
{int present, link_active;/** If the slot is on and presence or link has changed, turn it off.* Even if it's occupied again, we cannot assume the card is the same.*/mutex_lock(&ctrl->state_lock);switch (ctrl->state) {case BLINKINGOFF_STATE:cancel_delayed_work(&ctrl->button_work);fallthrough;case ON_STATE:ctrl->state = POWEROFF_STATE;mutex_unlock(&ctrl->state_lock);if (events & PCI_EXP_SLTSTA_DLLSC)ctrl_info(ctrl, "Slot(%s): Link Down\n",slot_name(ctrl));if (events & PCI_EXP_SLTSTA_PDC)ctrl_info(ctrl, "Slot(%s): Card not present\n",slot_name(ctrl));pciehp_disable_slot(ctrl, SURPRISE_REMOVAL);  //call remove_boardbreak;default:mutex_unlock(&ctrl->state_lock);break;}/* Turn the slot on if it's occupied or link is up */mutex_lock(&ctrl->state_lock);present = pciehp_card_present(ctrl);link_active = pciehp_check_link_active(ctrl);if (present <= 0 && link_active <= 0) {if (ctrl->state == BLINKINGON_STATE) {ctrl->state = OFF_STATE;cancel_delayed_work(&ctrl->button_work);pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF,INDICATOR_NOOP);ctrl_info(ctrl, "Slot(%s): Card not present\n",slot_name(ctrl));}mutex_unlock(&ctrl->state_lock);return;}switch (ctrl->state) {case BLINKINGON_STATE:cancel_delayed_work(&ctrl->button_work);fallthrough;case OFF_STATE:ctrl->state = POWERON_STATE;mutex_unlock(&ctrl->state_lock);if (present)ctrl_info(ctrl, "Slot(%s): Card present\n",slot_name(ctrl));if (link_active)ctrl_info(ctrl, "Slot(%s): Link Up\n",slot_name(ctrl));ctrl->request_result = pciehp_enable_slot(ctrl);  //call board_added()break;default:mutex_unlock(&ctrl->state_lock);break;}
}
pciehp_enable_slot会调用board_added（通过__pciehp_enable_slot）和pciehp_set_indicators
static int pciehp_enable_slot(struct controller *ctrl)
{int ret;pm_runtime_get_sync(&ctrl->pcie->port->dev);ret = __pciehp_enable_slot(ctrl);if (ret && ATTN_BUTTN(ctrl))/* may be blinking */pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF,INDICATOR_NOOP);pm_runtime_put(&ctrl->pcie->port->dev);mutex_lock(&ctrl->state_lock);ctrl->state = ret ? OFF_STATE : ON_STATE;mutex_unlock(&ctrl->state_lock);return ret;
}

board_added完成了pciehp_power_on_slot，pciehp_set_indicators，pciehp_configure_device (Off -> Blink -> On)

static int board_added(struct controller *ctrl)
{int retval = 0;struct pci_bus *parent = ctrl->pcie->port->subordinate;if (POWER_CTRL(ctrl)) {/* Power on slot */retval = pciehp_power_on_slot(ctrl);if (retval)return retval;}pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_BLINK,INDICATOR_NOOP);/* Check link training status */retval = pciehp_check_link_status(ctrl);if (retval)goto err_exit;/* Check for a power fault */if (ctrl->power_fault_detected || pciehp_query_power_fault(ctrl)) {ctrl_err(ctrl, "Slot(%s): Power fault\n", slot_name(ctrl));retval = -EIO;goto err_exit;}retval = pciehp_configure_device(ctrl);if (retval) {if (retval != -EEXIST) {ctrl_err(ctrl, "Cannot add device at %04x:%02x:00\n",pci_domain_nr(parent), parent->number);goto err_exit;}}pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_ON,PCI_EXP_SLTCTL_ATTN_IND_OFF);return 0;err_exit:set_slot_off(ctrl);return retval;
}

int pciehp_power_on_slot(struct controller *ctrl)
{struct pci_dev *pdev = ctrl_dev(ctrl);u16 slot_status;int retval;/* Clear power-fault bit from previous power failures */pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status);if (slot_status & PCI_EXP_SLTSTA_PFD)pcie_capability_write_word(pdev, PCI_EXP_SLTSTA,PCI_EXP_SLTSTA_PFD);ctrl->power_fault_detected = 0;pcie_write_cmd(ctrl, PCI_EXP_SLTCTL_PWR_ON, PCI_EXP_SLTCTL_PCC);ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL,PCI_EXP_SLTCTL_PWR_ON);retval = pciehp_link_enable(ctrl);if (retval)ctrl_err(ctrl, "%s: Can not enable the link!\n", __func__);return retval;
}

Surprise Remove Flow

需要处理AER和DPC events，然后处理hot-plug events。中断触发pciehp_handle_presence_or_link_change，然后pciehp_disable_slot会调用remove_board（通过__pciehp_disable_slot），remove_board主要完成pciehp_power_off_slot和pciehp_set_indicators (On -> Off)。Presense detected需要等待DPC先触发。

static int pciehp_disable_slot(struct controller *ctrl, bool safe_removal)
{int ret;pm_runtime_get_sync(&ctrl->pcie->port->dev);ret = __pciehp_disable_slot(ctrl, safe_removal);pm_runtime_put(&ctrl->pcie->port->dev);mutex_lock(&ctrl->state_lock);ctrl->state = OFF_STATE;mutex_unlock(&ctrl->state_lock);return ret;
}

static void remove_board(struct controller *ctrl, bool safe_removal)
{pciehp_unconfigure_device(ctrl, safe_removal);if (POWER_CTRL(ctrl)) {pciehp_power_off_slot(ctrl);/** After turning power off, we must wait for at least 1 second* before taking any action that relies on power having been* removed from the slot/adapter.*/msleep(1000);/* Ignore link or presence changes caused by power off */atomic_and(~(PCI_EXP_SLTSTA_DLLSC | PCI_EXP_SLTSTA_PDC),&ctrl->pending_events);}pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF,INDICATOR_NOOP);
}

void pciehp_power_off_slot(struct controller *ctrl)
{pcie_write_cmd(ctrl, PCI_EXP_SLTCTL_PWR_OFF, PCI_EXP_SLTCTL_PCC);ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL,PCI_EXP_SLTCTL_PWR_OFF);
}

NPEM Flow

NPEM(Native PCIe Enclosure Management) 是PCIe标准为NVMe设备定义的扩展LED控制功能，热插拔的LED控制机制由NPEM完成。可以实现在DP内或UP内，用于控制和更新LED状态。软件通过写入NPEM控制寄存器来发出NPEM指令，NPEM控制器根据指令更新LED状态，并使用completed机制指示软件完成。

NPEM支持实现OK/Locate/Fail/Rebuild等可选LED状态。

Linux Reference:
drivers/pci/hotplug /pciehp_hpc.c
driversdrivers/pci/hotplug/pciehp_ctrl.c