[cdi-devel] [PATCH 2/5] ehci: Implement interrupt transfers
Max Reitz ·
Signed-off-by: Max Reitz <max at tyndur.org> --- ehci/ehci.c | 265 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- ehci/ehci.h | 20 +++++ ehci/main.c | 11 +-- 3 files changed, 278 insertions(+), 18 deletions(-) diff --git a/ehci/ehci.c b/ehci/ehci.c index addec00..4b548ae 100644 --- a/ehci/ehci.c +++ b/ehci/ehci.c @@ -40,6 +40,13 @@ static void bios_handoff(struct cdi_pci_device *pci_dev, struct ehci *hc); static void irq_handler(struct cdi_device *dev); +static void init_periodic_pseudo_qhs(struct ehci *hc); + + +static inline bool is_power_of_two(unsigned value) +{ + return !(value & (value - 1)); +} struct cdi_device *ehci_init_device(struct cdi_bus_data *bus_data) @@ -111,6 +118,18 @@ struct cdi_device *ehci_init_device(struct cdi_bus_data *bus_data) } hc->regs->periodiclistbase = hc->periodic_list_mem->paddr.items[0].start; + hc->periodic_qhs = calloc(1024, sizeof(*hc->periodic_qhs)); + // 10 = |0..10| = |ld(1)..ld(1024)| + hc->periodic_pseudo_qhs_mem = + cdi_mem_alloc(11 * sizeof(hc->periodic_pseudo_qhs[0]), + 6 + | CDI_MEM_PHYS_CONTIGUOUS + | CDI_MEM_DMA_4G + | CDI_MEM_NOINIT); + hc->periodic_pseudo_qhs = hc->periodic_pseudo_qhs_mem->vaddr; + + init_periodic_pseudo_qhs(hc); + // Create anchor struct cdi_mem_area *anchor_qh; anchor_qh = cdi_mem_alloc(sizeof(ehci_fat_qh_t), 6 @@ -157,8 +176,8 @@ struct cdi_device *ehci_init_device(struct cdi_bus_data *bus_data) cdi_register_irq(pci_dev->irq, &irq_handler, &hc->hc.dev); - // async advance doorbell - hc->regs->usbintr = EHCI_INT_IAA; + // async advance doorbell, USB interrupt + hc->regs->usbintr = EHCI_INT_IAA | EHCI_INT_USBINT; // Route everything to this HC hc->regs->configflag |= EHCI_CF_CF; @@ -181,6 +200,10 @@ fail: if (hc->periodic_list_mem) { cdi_mem_free(hc->periodic_list_mem); } + free(hc->periodic_qhs); + if (hc->periodic_pseudo_qhs_mem) { + cdi_mem_free(hc->periodic_pseudo_qhs_mem); + } cdi_list_destroy(hc->retired_qhs); free((void *)hc->hc.dev.name); free(hc); @@ -223,6 +246,8 @@ static void bios_handoff(struct cdi_pci_device *pci_dev, struct ehci *hc) } +static void qtd_write_back(ehci_fat_qtd_t *qtd); + static void irq_handler(struct cdi_device *dev) { struct cdi_usb_hc *cdi_hc = CDI_UPCAST(dev, struct cdi_usb_hc, dev); @@ -231,14 +256,108 @@ static void irq_handler(struct cdi_device *dev) uint32_t usbsts = hc->regs->usbsts; hc->regs->usbsts = usbsts; - // async advance doorbell - if (!(usbsts & EHCI_STS_IAA)) { - return; + if (usbsts & EHCI_STS_IAA) { + // async advance doorbell + + ehci_fat_qh_t *rqh; + while ((rqh = cdi_list_pop(hc->retired_qhs))) { + cdi_mem_free(rqh->cdi_mem_area); + } } - ehci_fat_qh_t *rqh; - while ((rqh = cdi_list_pop(hc->retired_qhs))) { - cdi_mem_free(rqh->cdi_mem_area); + if (usbsts & EHCI_STS_USBINT) { + // IOC qTD retired + + for (ehci_fat_qh_t *qh = hc->periodic_functional_qhs; qh; + qh = qh->next_periodic) + { + if (qh->qh.overlay.status & EHCI_QTDS_ACTIVE) { + continue; + } + + bool error = false; + + for (ehci_fat_qtd_t *qtd = qh->first_qtd; qtd; qtd = qtd->next) { + assert((qtd->qtd.status & EHCI_QTDS_PID_MASK) == + EHCI_QTDS_PID_IN); + + error = error || (qtd->qtd.status & EHCI_QTDS_HALTED); + if (!error) { + qtd_write_back(qtd); + } + qtd->qtd.buffers[0] &= ~0xfffu; + qtd->qtd.status &= ~0x7fu; // clear errors + qtd->qtd.status |= EHCI_QTDS_XFLEN(qtd->write_back_size) + | EHCI_QTDS_IOC + | EHCI_QTDS_CP(0) + | EHCI_QTDS_CERR(3) + | EHCI_QTDS_PID_IN + | EHCI_QTDS_ACTIVE; + } + + if (!error) { + qh->cb(qh->cb_opaque); + } + + // Reactivate the QH + qh->qh.overlay.next_qtd = qh->first_qtd->paddr; + qh->qh.overlay.alt_next_qtd = qh->first_qtd->paddr; + qh->qh.overlay.status &= ~0x7fu; // clear errors + } + } +} + + +static void init_periodic_pseudo_qhs(struct ehci *hc) +{ + for (int e = 0; e < 11; e++) { + ehci_fat_qh_t *qh = &hc->periodic_pseudo_qhs[e]; + int interval = 1 << e; + + memset(qh, 0, sizeof(*qh)); + + qh->is_pseudo_qh = true; + + qh->paddr = hc->periodic_pseudo_qhs_mem->paddr.items[0].start + + e * sizeof(*qh); + + qh->qh.ep_state[0] = EHCI_QHES0_RL(0) + | EHCI_QHES0_MPL(0) + | EHCI_QHES0_DTC + | EHCI_QHES0_EP(0) + | EHCI_QHES0_DEV(0) + | EHCI_QHES0_EPS_HS; + qh->qh.ep_state[1] = EHCI_QHES1_MULT(1) + | EHCI_QHES1_PORT(0) + | EHCI_QHES1_HUB(0) + | EHCI_QHES1_SCM(0xff) + | EHCI_QHES1_ISM(0x01); + + qh->qh.overlay.next_qtd = EHCI_T; + qh->qh.overlay.alt_next_qtd = EHCI_T; + qh->qh.overlay.status = EHCI_QTDS_HALTED; + + + // EHCI specification, 4.6: "Interrupt queue heads are linked into the + // frame list ordered by poll rate. Longer poll rates are linked first + // [...]." + // Therefore, link from high exponents (intervals) to lower ones. + + if (e > 0) { + qh->qh.next_qh = hc->periodic_pseudo_qhs[e - 1].paddr | EHCI_QH; + qh->next = &hc->periodic_pseudo_qhs[e - 1]; + } else { + qh->qh.next_qh = EHCI_T; + } + + if (e < 10) { + qh->previous = &hc->periodic_pseudo_qhs[e + 1]; + } + + for (int i = 0; i < 1024; i += interval) { + hc->periodic_list[i] = qh->paddr | EHCI_QH; + hc->periodic_qhs[i] = qh; + } } } @@ -323,7 +442,28 @@ static void enter_async_qh(struct ehci *hc, ehci_fat_qh_t *qh) } -static void retire_qh(struct ehci *hc, ehci_fat_qh_t *qh) +static void enter_interrupt_qh(struct ehci *hc, ehci_fat_qh_t *qh, + int frame_interval_shift) +{ + qh->next = hc->periodic_functional_qhs; + hc->periodic_functional_qhs = qh; + + ehci_fat_qh_t *pseudo_qh = &hc->periodic_pseudo_qhs[frame_interval_shift]; + + qh->next = pseudo_qh->next; + qh->previous = pseudo_qh; + + pseudo_qh->next = qh; + if (qh->next) { + qh->next->previous = qh; + } + + qh->qh.next_qh = pseudo_qh->qh.next_qh; + pseudo_qh->qh.next_qh = qh->paddr | EHCI_QH; +} + + +static void retire_async_qh(struct ehci *hc, ehci_fat_qh_t *qh) { qh->previous->qh.next_qh = qh->qh.next_qh; @@ -336,11 +476,29 @@ static void retire_qh(struct ehci *hc, ehci_fat_qh_t *qh) } -static void retire_first_qtd(ehci_fat_qh_t *qh) +static void retire_periodic_qh(struct ehci *hc, ehci_fat_qh_t *qh) { - ehci_fat_qtd_t *qtd = qh->first_qtd; + qh->previous->qh.next_qh = qh->qh.next_qh; - assert(qtd); + qh->previous->next = qh->next; + if (qh->next) { + qh->next->previous = qh->previous; + } + + ehci_fat_qh_t **func_periodic_ptr = &hc->periodic_functional_qhs; + while (*func_periodic_ptr && *func_periodic_ptr != qh) { + func_periodic_ptr = &(*func_periodic_ptr)->next_periodic; + } + assert(*func_periodic_ptr == qh); + *func_periodic_ptr = qh->next_periodic; + + cdi_list_push(hc->retired_qhs, qh); +} + + +static void qtd_write_back(ehci_fat_qtd_t *qtd) +{ + ehci_fat_qh_t *qh = qtd->qh; uint32_t err_mask = (qh->qh.ep_state[0] & EHCI_QHES0_EPS_MASK) == EHCI_QHES0_EPS_HS @@ -371,6 +529,16 @@ static void retire_first_qtd(ehci_fat_qh_t *qh) } assert(!rem); } +} + + +static void retire_first_qtd(ehci_fat_qh_t *qh) +{ + ehci_fat_qtd_t *qtd = qh->first_qtd; + + assert(qtd); + + qtd_write_back(qtd); qh->first_qtd = qtd->next; if (qh->last_qtd == qtd) { @@ -561,6 +729,73 @@ void ehci_start_transaction(struct cdi_usb_hc *cdi_hc, } +void ehci_start_interrupt_transaction(struct cdi_usb_hc *cdi_hc, + cdi_usb_hc_transaction_t ta, + int interval, + cdi_usb_hc_interrupt_cb_t cb, + void *cb_opaque) +{ + struct ehci *hc = CDI_UPCAST(cdi_hc, struct ehci, hc); + ehci_fat_qh_t *qh = ta; + + // Set IOC for all qTDs + for (ehci_fat_qtd_t *qtd = qh->first_qtd; qtd; qtd = qtd->next) { + qtd->qtd.status |= EHCI_QTDS_IOC; + } + + assert(interval > 0 && is_power_of_two(interval)); + + assert(cb); + qh->cb = cb; + qh->cb_opaque = cb_opaque; + + if (interval > 1024 * 8) { + interval = 1024 * 8; + } + + // Moving the microframe index along looks like an easy way of balancing the + // periodic load a bit. + int start_microframe = hc->periodic_start_microframe; + hc->periodic_start_microframe = (hc->periodic_start_microframe + 1) % 8; + + bool split = + (qh->qh.ep_state[0] & EHCI_QHES0_EPS_MASK) != EHCI_QHES0_EPS_HS; + + if (split) { + // If split transactions start in the latter four microframes of a + // frame, their management is more difficult. Therefore, just limit it + // to the first four microframes. + start_microframe %= 4; + } + + int s_mask = 0; + + for (int microframe = start_microframe; microframe < start_microframe + 8; + microframe += interval) + { + s_mask |= 1 << (microframe % 8); + } + + assert(!(qh->qh.ep_state[1] & EHCI_QHES1_ISM(0xff))); + qh->qh.ep_state[1] |= EHCI_QHES1_ISM(s_mask); + + if (split) { + // Don't just issue complete split requests all over the place, but only + // when we actually should do so (3 requests starting 2 microframes + // after the start split request) + qh->qh.ep_state[1] &= ~EHCI_QHES1_SCM(0xff); + qh->qh.ep_state[1] |= EHCI_QHES1_SCM(7 << (start_microframe + 2)); + } + + int frame_interval_shift = __builtin_ctz(interval) - 3; + if (frame_interval_shift < 0) { + frame_interval_shift = 0; + } + + enter_interrupt_qh(hc, qh, frame_interval_shift); +} + + void ehci_wait_transaction(struct cdi_usb_hc *cdi_hc, cdi_usb_hc_transaction_t ta) { @@ -639,5 +874,9 @@ void ehci_destroy_transaction(struct cdi_usb_hc *cdi_hc, while (qh->first_qtd) { retire_first_qtd(qh); } - retire_qh(hc, qh); + if (qh->cb) { + retire_periodic_qh(hc, qh); + } else { + retire_async_qh(hc, qh); + } } diff --git a/ehci/ehci.h b/ehci/ehci.h index fb8dd9e..4b55391 100644 --- a/ehci/ehci.h +++ b/ehci/ehci.h @@ -87,8 +87,14 @@ struct ehci_fat_qh { struct cdi_mem_area *cdi_mem_area; uintptr_t paddr; + cdi_usb_hc_interrupt_cb_t cb; + void *cb_opaque; + ehci_fat_qh_t *next, *previous; + ehci_fat_qh_t *next_periodic; ehci_fat_qtd_t *first_qtd, *last_qtd; + + bool is_pseudo_qh; } __attribute__((aligned(64))); static_assert(!(sizeof(struct ehci_fat_qh) % 64), @@ -122,9 +128,17 @@ struct ehci { struct cdi_mem_area *periodic_list_mem; volatile uint32_t *periodic_list; + ehci_fat_qh_t **periodic_qhs; + + struct cdi_mem_area *periodic_pseudo_qhs_mem; + ehci_fat_qh_t *periodic_pseudo_qhs; + ehci_fat_qh_t *periodic_functional_qhs; ehci_fat_qh_t *async_start; cdi_list_t retired_qhs; + + // For a bit of load balancing + int periodic_start_microframe; }; #define PCI_CLASS_SERIAL_BUS_CONTROLLER 0x0c @@ -254,6 +268,7 @@ struct ehci { #define EHCI_QTDS_PID_OUT (0 << 8) // PID: OUT #define EHCI_QTDS_PID_IN (1 << 8) // PID: IN #define EHCI_QTDS_PID_SETUP (2 << 8) // PID: SETUP +#define EHCI_QTDS_PID_MASK (3 << 8) // PID mask #define EHCI_QTDS_ACTIVE (1 << 7) // Status: Active #define EHCI_QTDS_HALTED (1 << 6) // Status: Halted #define EHCI_QTDS_BUFERR (1 << 5) // Status: Data Buffer Error @@ -287,6 +302,11 @@ void ehci_enqueue(struct cdi_usb_hc *hc, void ehci_start_transaction(struct cdi_usb_hc *cdi_hc, cdi_usb_hc_transaction_t ta); void ehci_wait_transaction(struct cdi_usb_hc *hc, cdi_usb_hc_transaction_t ta); +void ehci_start_interrupt_transaction(struct cdi_usb_hc *hc, + cdi_usb_hc_transaction_t ta, + int interval, + cdi_usb_hc_interrupt_cb_t cb, + void *cb_opaque); void ehci_destroy_transaction(struct cdi_usb_hc *hc, cdi_usb_hc_transaction_t ta); diff --git a/ehci/main.c b/ehci/main.c index 5623819..97974ac 100644 --- a/ehci/main.c +++ b/ehci/main.c @@ -59,11 +59,12 @@ static struct cdi_usb_hcd ehcd = { .port_status = ehci_rh_port_status, }, - .create_transaction = ehci_create_transaction, - .enqueue = ehci_enqueue, - .start_transaction = ehci_start_transaction, - .wait_transaction = ehci_wait_transaction, - .destroy_transaction = ehci_destroy_transaction, + .create_transaction = ehci_create_transaction, + .enqueue = ehci_enqueue, + .start_transaction = ehci_start_transaction, + .wait_transaction = ehci_wait_transaction, + .start_interrupt_transaction = ehci_start_interrupt_transaction, + .destroy_transaction = ehci_destroy_transaction, }; CDI_DRIVER(DRIVER_NAME, ehcd) -- 2.6.4