Improved bcm2836/7 IRQ handling

drivers/irqchip/irq-bcm2835.c

@@ -152,7 +152,7 @@ static void armctrl_unmask_irq(struct irq_data *d)
 	}
 }
 
-#ifdef CONFIG_ARM64
+#if defined(CONFIG_SMP)
 void bcm2836_arm_irqchip_spin_gpu_irq(void);
 
 static void armctrl_ack_irq(struct irq_data *d)
@@ -166,7 +166,7 @@ static struct irq_chip armctrl_chip = {
 	.name = "ARMCTRL-level",
 	.irq_mask = armctrl_mask_irq,
 	.irq_unmask = armctrl_unmask_irq,
-#ifdef CONFIG_ARM64
+#if defined(CONFIG_SMP)
 	.irq_ack    = armctrl_ack_irq
 #endif
 };

drivers/irqchip/irq-bcm2836.c

@@ -87,19 +87,24 @@ static void bcm2836_arm_irqchip_unmask_gpu_irq(struct irq_data *d)
 {
 }
 
-#ifdef CONFIG_ARM64
+#if defined(CONFIG_SMP)
 
 void bcm2836_arm_irqchip_spin_gpu_irq(void)
 {
 	u32 i;
 	void __iomem *gpurouting = (intc.base + LOCAL_GPU_ROUTING);
 	u32 routing_val = readl(gpurouting);
+	u32 irq_route;
+
+	/* Preserve FIQ routing bits */
+	irq_route = routing_val & 0x3;
+	routing_val &= ~0x3;
 
 	for (i = 1; i <= 3; i++) {
-		u32 new_routing_val = (routing_val + i) & 3;
+		irq_route = (irq_route + i) & 0x3;
 
-		if (cpu_active(new_routing_val)) {
-			writel(new_routing_val, gpurouting);
+		if (cpu_active(irq_route)) {
+			writel(irq_route | routing_val, gpurouting);
 			return;
 		}
 	}

drivers/usb/host/dwc_otg/dwc_otg_fiq_fsm.c

@@ -74,70 +74,6 @@ void notrace _fiq_print(enum fiq_debug_level dbg_lvl, volatile struct fiq_state
 	}
 }
 
-
-#ifdef CONFIG_ARM64
-
-inline void fiq_fsm_spin_lock(fiq_lock_t *lock)
-{
-	spin_lock((spinlock_t *)lock);
-}
-
-inline void fiq_fsm_spin_unlock(fiq_lock_t *lock)
-{
-	spin_unlock((spinlock_t *)lock);
-}
-
-#else
-
-/**
- * fiq_fsm_spin_lock() - ARMv6+ bare bones spinlock
- * Must be called with local interrupts and FIQ disabled.
- */
-#if defined(CONFIG_ARCH_BCM2835) && defined(CONFIG_SMP)
-inline void fiq_fsm_spin_lock(fiq_lock_t *lock)
-{
-	unsigned long tmp;
-	uint32_t newval;
-	fiq_lock_t lockval;
-	/* Nested locking, yay. If we are on the same CPU as the fiq, then the disable
-	 * will be sufficient. If we are on a different CPU, then the lock protects us. */
-	prefetchw(&lock->slock);
-	asm volatile (
-	"1:     ldrex   %0, [%3]\n"
-	"       add     %1, %0, %4\n"
-	"       strex   %2, %1, [%3]\n"
-	"       teq     %2, #0\n"
-	"       bne     1b"
-	: "=&r" (lockval), "=&r" (newval), "=&r" (tmp)
-	: "r" (&lock->slock), "I" (1 << 16)
-	: "cc");
-
-	while (lockval.tickets.next != lockval.tickets.owner) {
-		wfe();
-		lockval.tickets.owner = READ_ONCE(lock->tickets.owner);
-	}
-	smp_mb();
-}
-#else
-inline void fiq_fsm_spin_lock(fiq_lock_t *lock) { }
-#endif
-
-/**
- * fiq_fsm_spin_unlock() - ARMv6+ bare bones spinunlock
- */
-#if defined(CONFIG_ARCH_BCM2835) && defined(CONFIG_SMP)
-inline void fiq_fsm_spin_unlock(fiq_lock_t *lock)
-{
-	smp_mb();
-	lock->tickets.owner++;
-	dsb_sev();
-}
-#else
-inline void fiq_fsm_spin_unlock(fiq_lock_t *lock) { }
-#endif
-
-#endif
-
 /**
  * fiq_fsm_restart_channel() - Poke channel enable bit for a split transaction
  * @channel: channel to re-enable

drivers/usb/host/dwc_otg/dwc_otg_fiq_fsm.h

@@ -129,12 +129,7 @@
 	FIQDBG_PORTHUB = (1 << 3),
 };
 
-#ifdef CONFIG_ARM64
-
-typedef spinlock_t fiq_lock_t;
-
-#else
-
+#define TICKET_SHIFT 16
 typedef struct {
 	union {
 		uint32_t slock;
@@ -143,7 +138,97 @@
 			uint16_t next;
 		} tickets;
 	};
-} fiq_lock_t;
+} __aligned(4) fiq_lock_t;
+
+#if defined(CONFIG_ARM64)
+static inline void fiq_fsm_spin_lock(fiq_lock_t *lock)
+{
+	unsigned int tmp;
+	fiq_lock_t lockval, newval;
+
+	asm volatile(
+	/* Atomically increment the next ticket. */
+	"	prfm	pstl1strm, %3\n"
+	"1:	ldaxr	%w0, %3\n"
+	"	add	%w1, %w0, %w5\n"
+	"	stxr	%w2, %w1, %3\n"
+	"	cbnz	%w2, 1b\n"
+	/* Did we get the lock? */
+	"	eor	%w1, %w0, %w0, ror #16\n"
+	"	cbz	%w1, 3f\n"
+	/*
+	 * No: spin on the owner. Send a local event to avoid missing an
+	 * unlock before the exclusive load.
+	 */
+	"	sevl\n"
+	"2:	wfe\n"
+	"	ldaxrh	%w2, %4\n"
+	"	eor	%w1, %w2, %w0, lsr #16\n"
+	"	cbnz	%w1, 2b\n"
+	/* We got the lock. Critical section starts here. */
+	"3:"
+	: "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock)
+	: "Q" (lock->tickets.owner), "I" (1 << TICKET_SHIFT)
+	: "memory");
+}
+
+static inline void fiq_fsm_spin_unlock(fiq_lock_t *lock)
+{
+	asm volatile(
+	"	stlrh	%w1, %0\n"
+	: "=Q" (lock->tickets.owner)
+	: "r" (lock->tickets.owner + 1)
+	: "memory");
+}
+
+#else
+
+/**
+ * fiq_fsm_spin_lock() - ARMv6+ bare bones spinlock
+ * Must be called with local interrupts and FIQ disabled.
+ */
+#if defined(CONFIG_ARCH_BCM2835) && defined(CONFIG_SMP)
+static inline void fiq_fsm_spin_lock(fiq_lock_t *lock)
+{
+	unsigned long tmp;
+	uint32_t newval;
+	fiq_lock_t lockval;
+	/* Nested locking, yay. If we are on the same CPU as the fiq, then the disable
+	 * will be sufficient. If we are on a different CPU, then the lock protects us. */
+	prefetchw(&lock->slock);
+	asm volatile (
+	"1:     ldrex   %0, [%3]\n"
+	"       add     %1, %0, %4\n"
+	"       strex   %2, %1, [%3]\n"
+	"       teq     %2, #0\n"
+	"       bne     1b"
+	: "=&r" (lockval), "=&r" (newval), "=&r" (tmp)
+	: "r" (&lock->slock), "I" (1 << TICKET_SHIFT)
+	: "cc");
+
+	while (lockval.tickets.next != lockval.tickets.owner) {
+		wfe();
+		lockval.tickets.owner = READ_ONCE(lock->tickets.owner);
+	}
+	smp_mb();
+}
+#else
+static inline void fiq_fsm_spin_lock(fiq_lock_t *lock) { }
+#endif
+
+/**
+ * fiq_fsm_spin_unlock() - ARMv6+ bare bones spinunlock
+ */
+#if defined(CONFIG_ARCH_BCM2835) && defined(CONFIG_SMP)
+static inline void fiq_fsm_spin_unlock(fiq_lock_t *lock)
+{
+	smp_mb();
+	lock->tickets.owner++;
+	dsb_sev();
+}
+#else
+static inline void fiq_fsm_spin_unlock(fiq_lock_t *lock) { }
+#endif
 
 #endif
 
@@ -380,10 +465,6 @@
 
 #endif
 
-extern void fiq_fsm_spin_lock(fiq_lock_t *lock);
-
-extern void fiq_fsm_spin_unlock(fiq_lock_t *lock);
-
 extern int fiq_fsm_too_late(struct fiq_state *st, int n);
 
 extern int fiq_fsm_tt_in_use(struct fiq_state *st, int num_channels, int n);

drivers/usb/host/dwc_otg/dwc_otg_hcd.c

@@ -1042,10 +1042,6 @@
 		}
 		DWC_MEMSET(hcd->fiq_state, 0, (sizeof(struct fiq_state) + (sizeof(struct fiq_channel_state) * num_channels)));
 
-#ifdef CONFIG_ARM64
-		spin_lock_init(&hcd->fiq_state->lock);
-#endif
-
 		hcd->fiq_state->dummy_send = DWC_DMA_ALLOC_ATOMIC(dev, 16,
 							 &hcd->fiq_state->dummy_send_dma);
 
@@ -1067,7 +1063,7 @@
 		 * moderately readable array casts.
 		 */
 		hcd->fiq_dmab = DWC_DMA_ALLOC(dev, (sizeof(struct fiq_dma_channel) * num_channels), &hcd->fiq_state->dma_base);
-		DWC_WARN("FIQ DMA bounce buffers: virt = %px dma = %pad len=%zu",
+		DWC_INFO("FIQ DMA bounce buffers: virt = %px dma = %pad len=%zu",
 				hcd->fiq_dmab, &hcd->fiq_state->dma_base,
 				sizeof(struct fiq_dma_channel) * num_channels);
 

drivers/usb/host/dwc_otg/dwc_otg_hcd_linux.c

@@ -406,19 +406,17 @@
 
 #ifdef CONFIG_ARM64
 
-static int simfiq_irq = -1;
-
-void local_fiq_enable(void)
-{
-	if (simfiq_irq >= 0)
-		enable_irq(simfiq_irq);
-}
+/*
+ * With no FIQ support on AARCH64, the "FIQ handler" is demoted to a
+ * regular IRQ handler. With a nested spinlock preventing the two
+ * handlers from racing against each other, and a HCD lock preventing
+ * thread context from racing against the "bottom half" IRQ, there's no
+ * point manipulating global IRQ enable/disable state - so these two
+ * functions are no-ops.
+ */
+void local_fiq_enable(void) { }
 
-void local_fiq_disable(void)
-{
-	if (simfiq_irq >= 0)
-		disable_irq(simfiq_irq);
-}
+void local_fiq_disable(void) { }
 
 irqreturn_t fiq_irq_handler(int irq, void *dev_id)
 {
@@ -454,10 +452,10 @@
 		DWC_ERROR("Can't claim FIQ");
 		BUG();
 	}
-	DWC_WARN("FIQ on core %d", smp_processor_id());
-	DWC_WARN("FIQ ASM at %px length %d", &_dwc_otg_fiq_stub, (int)(&_dwc_otg_fiq_stub_end - &_dwc_otg_fiq_stub));
+	DWC_INFO("FIQ on core %d", smp_processor_id());
+	DWC_INFO("FIQ ASM at %px length %d", &_dwc_otg_fiq_stub, (int)(&_dwc_otg_fiq_stub_end - &_dwc_otg_fiq_stub));
 	set_fiq_handler((void *) &_dwc_otg_fiq_stub, &_dwc_otg_fiq_stub_end - &_dwc_otg_fiq_stub);
 	memset(&regs,0,sizeof(regs));
 
 	regs.ARM_r8 = (long) dwc_otg_hcd->fiq_state;
 	if (fiq_fsm_enable) {
@@ -482,9 +480,9 @@
 			otg_dev->os_dep.mphi_base + 0x1f0;
 		dwc_otg_hcd->fiq_state->mphi_regs.swirq_clr =
 			otg_dev->os_dep.mphi_base + 0x1f4;
-		DWC_WARN("Fake MPHI regs_base at %px",
+		DWC_INFO("Fake MPHI regs_base at %px",
 			 dwc_otg_hcd->fiq_state->mphi_regs.base);
 	} else {
 		dwc_otg_hcd->fiq_state->mphi_regs.ctrl =
 			otg_dev->os_dep.mphi_base + 0x4c;
 		dwc_otg_hcd->fiq_state->mphi_regs.outdda
@@ -493,16 +491,16 @@
 			= otg_dev->os_dep.mphi_base + 0x2c;
 		dwc_otg_hcd->fiq_state->mphi_regs.intstat
 			= otg_dev->os_dep.mphi_base + 0x50;
-		DWC_WARN("MPHI regs_base at %px",
+		DWC_INFO("MPHI regs_base at %px",
 			 dwc_otg_hcd->fiq_state->mphi_regs.base);
 
 		//Enable mphi peripheral
 		writel((1<<31),dwc_otg_hcd->fiq_state->mphi_regs.ctrl);
 #ifdef DEBUG
 		if (readl(dwc_otg_hcd->fiq_state->mphi_regs.ctrl) & 0x80000000)
-			DWC_WARN("MPHI periph has been enabled");
+			DWC_INFO("MPHI periph has been enabled");
 		else
-			DWC_WARN("MPHI periph has NOT been enabled");
+			DWC_INFO("MPHI periph has NOT been enabled");
 #endif
 	}
 	// Enable FIQ interrupt from USB peripheral
@@ -521,7 +519,6 @@
 		return;
 	}
 
-	simfiq_irq = irq;
 #else
 #ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER
 	irq = otg_dev->os_dep.fiq_num;