Index: conf/files.i386 =========================================================================== --- sys/conf/files.i386 2003/10/30 14:48:07 #88 +++ conf/files.i386 2003/10/30 14:48:07 @@ -167,12 +167,14 @@ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.h acpi_wakecode.o acpi_wakecode.bin" # +i386/acpica/madt.c optional acpi apic i386/bios/apm.c optional apm i386/bios/mca_machdep.c optional mca i386/bios/smapi.c optional smapi i386/bios/smapi_bios.S optional smapi i386/bios/smbios.c optional smbios i386/bios/vpd.c optional vpd +i386/i386/apic_vector.s optional apic i386/i386/atomic.c standard \ compile-with "${CC} -c ${CFLAGS} ${DEFINED_PROF:S/^$/-fomit-frame-pointer/} ${.IMPSRC}" i386/i386/autoconf.c standard @@ -193,15 +195,20 @@ i386/i386/identcpu.c standard i386/i386/in_cksum.c optional inet i386/i386/initcpu.c standard +i386/i386/intr_machdep.c standard +i386/i386/io_apic.c optional apic i386/i386/k6_mem.c standard i386/i386/legacy.c standard +i386/i386/local_apic.c optional apic i386/i386/locore.s standard no-obj i386/i386/machdep.c standard i386/i386/mem.c standard i386/i386/mp_clock.c optional smp i386/i386/mp_machdep.c optional smp -i386/i386/mpapic.c optional smp i386/i386/mpboot.s optional smp +i386/i386/mptable.c optional smp +i386/i386/mptable.c optional apic +i386/i386/mptable_pci.c optional apic pci i386/i386/nexus.c standard i386/i386/perfmon.c optional perfmon i386/i386/perfmon.c optional perfmon profiling-routine @@ -233,6 +240,8 @@ i386/ibcs2/ibcs2_xenix_sysent.c optional ibcs2 i386/ibcs2/imgact_coff.c optional ibcs2 i386/isa/asc.c count asc +i386/isa/atpic.c standard +i386/isa/atpic_vector.s standard i386/isa/clock.c standard i386/isa/cronyx.c optional cx i386/isa/ctx.c optional ctx @@ -246,8 +255,6 @@ i386/isa/if_el.c count el i386/isa/if_le.c count le i386/isa/if_rdp.c count rdp -i386/isa/intr_machdep.c standard -i386/isa/ithread.c standard i386/isa/isa.c optional isa i386/isa/isa_compat.c optional isa compat_oldisa \ warning "Old ISA driver compatibility shims present." @@ -255,6 +262,7 @@ i386/isa/istallion.c optional stli nowerror i386/isa/loran.c optional loran i386/isa/mse.c optional mse +i386/isa/nmi.c standard i386/isa/npx.c optional npx i386/isa/pcf.c optional pcf i386/isa/pcvt/pcvt_drv.c optional vt Index: conf/options.i386 =========================================================================== --- sys/conf/options.i386 2003/10/30 14:48:07 #49 +++ conf/options.i386 2003/10/30 14:48:07 @@ -7,6 +7,7 @@ COMPAT_OLDISA I586_PMC_GUPROF opt_i586_guprof.h MAXMEM +NO_MIXED_MODE PERFMON DISABLE_PSE opt_pmap.h DISABLE_PG_G opt_pmap.h @@ -25,8 +26,6 @@ DEBUG_SVR4 opt_svr4.h PECOFF_SUPPORT opt_dontuse.h PECOFF_DEBUG opt_pecoff.h -# i386 SMP options -APIC_IO opt_global.h # Change KVM size. Changes things all over the kernel. KVA_PAGES opt_global.h @@ -146,6 +145,7 @@ IPR_LOG opt_i4b.h # Device options +DEV_APIC opt_apic.h DEV_NPX opt_npx.h # ------------------------------- Index: contrib/dev/acpica/tbxfroot.c =========================================================================== --- sys/contrib/dev/acpica/tbxfroot.c 2003/10/30 14:48:07 #12 +++ contrib/dev/acpica/tbxfroot.c 2003/10/30 14:48:07 @@ -420,6 +420,8 @@ return_ACPI_STATUS (AE_NO_ACPI_TABLES); } + /* XXXTEST */ + printf("%s: found RSDP at 0x%jx\n", __func__, TableInfo.PhysicalAddress); RsdpAddress->PointerType = ACPI_PHYSICAL_POINTER; RsdpAddress->Pointer.Physical = TableInfo.PhysicalAddress; return_ACPI_STATUS (AE_OK); @@ -522,6 +524,7 @@ */ Status = AcpiOsMapMemory ((UINT64) ACPI_LO_RSDP_WINDOW_BASE, ACPI_LO_RSDP_WINDOW_SIZE, (void **) &TablePtr); + printf("%s: Map(%jx, %u) = %p\n", __func__, (uintmax_t)ACPI_LO_RSDP_WINDOW_BASE, ACPI_LO_RSDP_WINDOW_SIZE, TablePtr); if (ACPI_FAILURE (Status)) { ACPI_DEBUG_PRINT ((ACPI_DB_ERROR, "Could not map memory at %X for length %X\n", @@ -538,6 +541,7 @@ PhysAddr = ACPI_LO_RSDP_WINDOW_BASE; PhysAddr += ACPI_PTR_DIFF (MemRover,TablePtr); + printf("%s: Found at %p, PA = %jx\n", __func__, MemRover, (uintmax_t)PhysAddr); TableInfo->PhysicalAddress = PhysAddr; return_ACPI_STATUS (AE_OK); @@ -548,6 +552,7 @@ */ Status = AcpiOsMapMemory ((UINT64) ACPI_HI_RSDP_WINDOW_BASE, ACPI_HI_RSDP_WINDOW_SIZE, (void **) &TablePtr); + printf("%s: Map(%jx, %u) = %p\n", __func__, (uintmax_t)ACPI_HI_RSDP_WINDOW_BASE, ACPI_HI_RSDP_WINDOW_SIZE, TablePtr); if (ACPI_FAILURE (Status)) { ACPI_DEBUG_PRINT ((ACPI_DB_ERROR, "Could not map memory at %X for length %X\n", @@ -565,6 +570,7 @@ PhysAddr = ACPI_HI_RSDP_WINDOW_BASE; PhysAddr += ACPI_PTR_DIFF (MemRover, TablePtr); + printf("%s: Found at %p, PA = %jx\n", __func__, MemRover, (uintmax_t)PhysAddr); TableInfo->PhysicalAddress = PhysAddr; return_ACPI_STATUS (AE_OK); } Index: dev/pci/pci.c =========================================================================== --- sys/dev/pci/pci.c 2003/10/30 14:48:07 #56 +++ dev/pci/pci.c 2003/10/30 14:48:07 @@ -820,7 +820,7 @@ } if (cfg->intpin > 0 && PCI_INTERRUPT_VALID(cfg->intline)) { -#if defined(__ia64__) || (defined(__i386__) && !defined(SMP)) +#if defined(__ia64__) || defined(__i386__) /* * Try to re-route interrupts. Sometimes the BIOS or * firmware may leave bogus values in these registers. Index: dev/sio/sio.c =========================================================================== --- sys/dev/sio/sio.c 2003/10/30 14:48:07 #67 +++ dev/sio/sio.c 2003/10/30 14:48:07 @@ -1161,6 +1161,9 @@ rid = 0; com->irqres = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, 0ul, ~0ul, 1, RF_ACTIVE); + if (com->irqres == NULL) + com->irqres = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, 0ul, + ~0ul, 1, RF_ACTIVE | RF_SHAREABLE); if (com->irqres) { ret = BUS_SETUP_INTR(device_get_parent(dev), dev, com->irqres, INTR_TYPE_TTY | INTR_FAST, Index: i386/acpica/acpi_wakeup.c =========================================================================== --- sys/i386/acpica/acpi_wakeup.c 2003/10/30 14:48:07 #28 +++ i386/acpica/acpi_wakeup.c 2003/10/30 14:48:07 @@ -47,10 +47,9 @@ #include #include +#include #include -#include - #include "acpi.h" #include @@ -222,6 +221,8 @@ ret_addr = 0; if (acpi_savecpu()) { /* Execute Sleep */ + intr_suspend(); + p_gdt = (struct region_descriptor *) (sc->acpi_wakeaddr + physical_gdt); p_gdt->rd_limit = r_gdt.rd_limit; @@ -271,7 +272,7 @@ #if 0 initializecpu(); #endif - icu_reinit(); + intr_resume(); if (acpi_get_verbose(sc)) { acpi_savecpu(); Index: i386/acpica/madt.c =========================================================================== *** /dev/null Thu Oct 30 14:47:44 2003 --- i386/acpica/madt.c Thu Oct 30 14:48:08 2003 *************** *** 0 **** --- 1,641 ---- + /*- + * Copyright (c) 2003 John Baldwin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + #include + #include + #include + #include + #include + + #include + #include + #include + + #include + #include + #include + #include + #include + #include + + #include "acpi.h" + #include + #include + #include + + #define NIOAPICS 16 /* Max number of I/O APICs */ + #define NLAPICS 16 /* Max number of local APICs */ + + typedef void madt_entry_handler(APIC_HEADER *entry, void *arg); + + /* These two arrays are indexed by APIC IDs. */ + struct ioapic_info { + void *io_apic; + UINT32 io_vector; + } ioapics[NIOAPICS]; + + struct lapic_info { + int la_present:1; + int la_enabled:1; + int la_apic_id:8; + } lapics[NLAPICS + 1]; + + static APIC_TABLE *madt; + static vm_paddr_t madt_physaddr; + static vm_offset_t madt_length; + + MALLOC_DEFINE(M_MADT, "MADT Table", "ACPI MADT Table Items"); + + static u_char interrupt_polarity(UINT16 Polarity); + static u_char interrupt_trigger(UINT16 TriggerMode); + static int madt_find_cpu(u_int acpi_id, u_int *apic_id); + static int madt_find_interrupt(int intr, void **apic, u_int *pin); + static void *madt_map(vm_paddr_t pa, int offset, vm_offset_t length); + static void *madt_map_table(vm_paddr_t pa, int offset, const char *sig); + static void madt_parse_apics(APIC_HEADER *entry, void *arg); + static void madt_parse_interrupt_override(INTERRUPT_SOURCE_OVERRIDE *intr); + static void madt_parse_ints(APIC_HEADER *entry, void *arg __unused); + static void madt_parse_local_nmi(LAPIC_NMI *nmi); + static void madt_parse_nmi(NMI *nmi); + static int madt_probe(void); + static int madt_probe_cpus(void); + static void madt_probe_cpus_handler(APIC_HEADER *entry, void *arg __unused); + static int madt_probe_table(vm_paddr_t address); + static void madt_register(void *dummy); + static int madt_setup_local(void); + static int madt_setup_io(void); + static void madt_unmap(void *data, vm_offset_t length); + static void madt_unmap_table(void *table); + static void madt_walk_table(madt_entry_handler *handler, void *arg); + + static struct apic_enumerator madt_enumerator = { + "MADT", + madt_probe, + madt_probe_cpus, + madt_setup_local, + madt_setup_io + }; + + /* + * Code to abuse the crashdump map to map in the tables for the early + * probe. We cheat and make the following assumptions about how we + * use this KVA: page 0 is used to map in the first page of each table + * found via the RSDT or XSDT and pages 1 to n are used to map in the + * RSDT or XSDT. The offset is in pages; the length is in bytes. + */ + static void * + madt_map(vm_paddr_t pa, int offset, vm_offset_t length) + { + vm_offset_t va, off; + void *data; + + off = pa & PAGE_MASK; + length = roundup(length + off, PAGE_SIZE); + pa = pa & PG_FRAME; + va = (vm_offset_t)pmap_kenter_temporary(pa, offset) + + (offset * PAGE_SIZE); + data = (void *)(va + off); + length -= PAGE_SIZE; + while (length > 0) { + va += PAGE_SIZE; + pa += PAGE_SIZE; + length -= PAGE_SIZE; + pmap_kenter(va, pa); + invlpg(va); + } + return (data); + } + + static void + madt_unmap(void *data, vm_offset_t length) + { + vm_offset_t va, off; + + va = (vm_offset_t)data; + off = va & PAGE_MASK; + length = roundup(length + off, PAGE_SIZE); + va &= ~PAGE_MASK; + while (length > 0) { + pmap_kremove(va); + invlpg(va); + va += PAGE_SIZE; + length -= PAGE_SIZE; + } + } + + static void * + madt_map_table(vm_paddr_t pa, int offset, const char *sig) + { + ACPI_TABLE_HEADER *header; + vm_offset_t length; + + header = madt_map(pa, offset, sizeof(ACPI_TABLE_HEADER)); + if (strncmp(header->Signature, sig, 4) != 0) { + madt_unmap(header, sizeof(ACPI_TABLE_HEADER)); + return (NULL); + } + length = header->Length; + madt_unmap(header, sizeof(ACPI_TABLE_HEADER)); + return (madt_map(pa, offset, length)); + } + + static void + madt_unmap_table(void *table) + { + ACPI_TABLE_HEADER *header; + + header = (ACPI_TABLE_HEADER *)table; + madt_unmap(table, header->Length); + } + + /* + * Look for an ACPI Multiple APIC Description Table ("APIC") + */ + static int + madt_probe(void) + { + ACPI_POINTER rsdp_ptr; + RSDP_DESCRIPTOR *rsdp; + RSDT_DESCRIPTOR *rsdt; + XSDT_DESCRIPTOR *xsdt; + int i, count; + + if (resource_disabled("acpi", 0)) + return (ENXIO); + + /* + * Map in the RSDP. Since ACPI uses AcpiOsMapMemory() which in turn + * calls pmap_mapdev() to find the RSDP, we assume that we can use + * pmap_mapdev() to map the RSDP. + */ + if (AcpiOsGetRootPointer(ACPI_LOGICAL_ADDRESSING, &rsdp_ptr) != AE_OK) + return (ENXIO); + /* XXXTEST */ + printf("rsdp_ptr.Pointer.Logical = %p\n", + (void *)rsdp_ptr.Pointer.Logical); + printf("rsdp_ptr.Pointer.Physical = 0x%jx\n", + (uintmax_t)rsdp_ptr.Pointer.Physical); + KASSERT(rsdp_ptr.Pointer.Physical < KERNLOAD, ("RSDP too high")); + rsdp = pmap_mapdev(rsdp_ptr.Pointer.Physical, sizeof(RSDP_DESCRIPTOR)); + if (rsdp == NULL) { + #if 0 + if (bootverbose) + #endif + printf("MADT: Failed to map RSDP\n"); + return (ENXIO); + } + + /* + * For ACPI < 2.0, use the RSDT. For ACPI >= 2.0, use the XSDT. + * We map the XSDT and RSDT at page 1 in the crashdump area. + * Page 0 is used to map in the headers of candidate ACPI tables. + */ + if (rsdp->Revision >= 2) { + /* XXXTEST */ + printf("XSDT PA = 0x%jx\n", (uintmax_t)rsdp->XsdtPhysicalAddress); + xsdt = madt_map_table(rsdp->XsdtPhysicalAddress, 1, XSDT_SIG); + if (xsdt == NULL) { + #if 0 + if (bootverbose) + #endif + printf("MADT: Failed to map XSDT\n"); + return (ENXIO); + } + count = (xsdt->Header.Length - sizeof(ACPI_TABLE_HEADER)) / + sizeof(UINT64); + for (i = 0; i < count; i++) + if (madt_probe_table(xsdt->TableOffsetEntry[i])) + break; + madt_unmap_table(xsdt); + } else { + /* XXXTEST */ + printf("RSDT PA = 0x%jx\n", (uintmax_t)rsdp->RsdtPhysicalAddress); + rsdt = madt_map_table(rsdp->RsdtPhysicalAddress, 1, RSDT_SIG); + if (rsdt == NULL) { + #if 0 + if (bootverbose) + #endif + printf("MADT: Failed to map RSDT\n"); + return (ENXIO); + } + count = (rsdt->Header.Length - sizeof(ACPI_TABLE_HEADER)) / + sizeof(UINT32); + for (i = 0; i < count; i++) + if (madt_probe_table(rsdt->TableOffsetEntry[i])) + break; + madt_unmap_table(rsdt); + } + pmap_unmapdev((vm_offset_t)rsdp, sizeof(RSDP_DESCRIPTOR)); + if (madt_physaddr == 0) { + #if 0 + if (bootverbose) + #endif + printf("MADT: No MADT table found\n"); + return (ENXIO); + } + #if 0 + if (bootverbose) + #endif + printf("MADT: Found table at %p\n", (void *)madt_physaddr); + + return (0); + } + + /* + * See if a given ACPI table is the MADT. + */ + static int + madt_probe_table(vm_paddr_t address) + { + ACPI_TABLE_HEADER *table; + + table = madt_map(address, 0, sizeof(ACPI_TABLE_HEADER)); + if (table == NULL) { + #if 0 + if (bootverbose) + #endif + printf("MADT: Failed to map table at %p\n", + (void *)address); + return (0); + } + #if 0 + if (bootverbose) + #endif + printf("Table '%.4s' at %p\n", table->Signature, + (void *)address); + + /* XXX: Verify checksum? */ + if (strncmp(table->Signature, APIC_SIG, 4) != 0) { + madt_unmap(table, sizeof(ACPI_TABLE_HEADER)); + return (0); + } + madt_physaddr = address; + madt_length = table->Length; + madt_unmap(table, sizeof(ACPI_TABLE_HEADER)); + return (1); + } + + /* + * Run through the MP table enumerating CPUs. + */ + static int + madt_probe_cpus(void) + { + + madt = madt_map_table(madt_physaddr, 0, APIC_SIG); + KASSERT(madt != NULL, ("Unable to re-map MADT")); + madt_walk_table(madt_probe_cpus_handler, NULL); + madt_unmap_table(madt); + madt = NULL; + return (0); + } + + /* + * Initialize the local APIC on the BSP. + */ + static int + madt_setup_local(void) + { + + madt = pmap_mapdev(madt_physaddr, madt_length); + lapic_init((uintptr_t)madt->LocalApicAddress); + printf("ACPI APIC Table: <%.*s %.*s>\n", + sizeof(madt->Header.OemId), madt->Header.OemId, + sizeof(madt->Header.OemTableId), madt->Header.OemTableId); + + /* + * We ignore 64-bit local APIC override entries. Should we + * perhaps emit a warning here if we find one? + */ + return (0); + } + + /* + * Run through the MP table enumerating I/O APICs. + */ + static int + madt_setup_io(void) + { + int i; + + /* First, we run through adding I/O APIC's. */ + madt_walk_table(madt_parse_apics, NULL); + + /* Second, we run through the table tweaking interrupt sources. */ + madt_walk_table(madt_parse_ints, NULL); + + /* Third, we register all the I/O APIC's. */ + for (i = 0; i < NIOAPICS; i++) + if (ioapics[i].io_apic != NULL) + ioapic_register(ioapics[i].io_apic); + + /* Finally, we throw the switch to enable the I/O APIC's. */ + acpi_SetDefaultIntrModel(ACPI_INTR_APIC); + + return (0); + } + + static void + madt_register(void *dummy __unused) + { + + apic_register_enumerator(&madt_enumerator); + } + SYSINIT(madt_register, SI_SUB_TUNABLES - 1, SI_ORDER_FIRST, + madt_register, NULL) + + /* + * Call the handler routine for each entry in the MADT table. + */ + static void + madt_walk_table(madt_entry_handler *handler, void *arg) + { + APIC_HEADER *entry; + u_char *p, *end; + + end = (u_char *)(madt) + madt->Header.Length; + for (p = (u_char *)(madt + 1); p < end; ) { + entry = (APIC_HEADER *)p; + handler(entry, arg); + p += entry->Length; + } + } + + static void + madt_probe_cpus_handler(APIC_HEADER *entry, void *arg) + { + PROCESSOR_APIC *proc; + struct lapic_info *la; + + switch (entry->Type) { + case APIC_PROC: + /* + * The MADT does not include a BSP flag, so we have to + * let the MP code figure out which CPU is the BSP on + * its own. + */ + proc = (PROCESSOR_APIC *)entry; + #if 0 + if (bootverbose) + #endif + printf("MADT: Found CPU APIC ID %d ACPI ID %d: %s\n", + proc->LocalApicId, proc->ProcessorApicId, + proc->ProcessorEnabled ? "enabled" : "disabled"); + if (proc->ProcessorApicId > NLAPICS) + panic("%s: CPU ID %d too high", __func__, + proc->ProcessorApicId); + la = &lapics[proc->ProcessorApicId]; + KASSERT(la->la_present == 0, + ("Duplicate local ACPI ID %d", proc->ProcessorApicId)); + la->la_present = 1; + la->la_apic_id = proc->LocalApicId; + if (proc->ProcessorEnabled) { + la->la_enabled = 1; + lapic_create(proc->LocalApicId, 0); + } + break; + } + } + + + /* + * Add an I/O APIC from an entry in the table. + */ + static void + madt_parse_apics(APIC_HEADER *entry, void *arg __unused) + { + IO_APIC *apic; + + + switch (entry->Type) { + case APIC_IO: + apic = (IO_APIC *)entry; + #if 0 + if (bootverbose) + #endif + printf("MADT: Found IO APIC ID %d, Vector %d at %p\n", + apic->IoApicId, apic->Vector, + (void *)apic->IoApicAddress); + if (apic->IoApicId >= NIOAPICS) + panic("%s: I/O APIC ID %d too high", __func__, + apic->IoApicId); + if (ioapics[apic->IoApicId].io_apic != NULL) + panic("%s: Double APIC ID %d", __func__, + apic->IoApicId); + ioapics[apic->IoApicId].io_apic = ioapic_create( + (uintptr_t)apic->IoApicAddress, apic->IoApicId, + apic->Vector); + ioapics[apic->IoApicId].io_vector = apic->Vector; + break; + default: + break; + } + } + + /* + * Determine properties of an interrupt source. Note that for ACPI, + * these are only used for ISA interrupts, so we assume ISA bus values + * (Active Hi, Edge Triggered) for conforming values. + */ + static u_char + interrupt_polarity(UINT16 Polarity) + { + + switch (Polarity) { + case APIC_POLARITY_CONFORM: + case APIC_POLARITY_ACTIVEHI: + return (1); + case APIC_POLARITY_ACTIVELO: + return (0); + default: + panic("Bogus Interrupt Polarity"); + } + } + + static u_char + interrupt_trigger(UINT16 TriggerMode) + { + + switch (TriggerMode) { + case APIC_TRIGGER_CONFORM: + case APIC_TRIGGER_EDGE: + return (1); + case APIC_TRIGGER_LEVEL: + return (0); + default: + panic("Bogus Interrupt Trigger Mode"); + } + } + + /* + * Find the local APIC ID associated with a given ACPI Processor ID. + */ + static int + madt_find_cpu(u_int acpi_id, u_int *apic_id) + { + + if (!lapics[acpi_id].la_present) + return (ENOENT); + *apic_id = lapics[acpi_id].la_apic_id; + if (lapics[acpi_id].la_enabled) + return (0); + else + return (ENXIO); + } + + /* + * Find the IO APIC and pin on that APIC associated with a given global + * interrupt. + */ + static int + madt_find_interrupt(int intr, void **apic, u_int *pin) + { + int i, best; + + best = -1; + for (i = 0; i < NIOAPICS; i++) { + if (ioapics[i].io_apic == NULL || + ioapics[i].io_vector > intr) + continue; + if (best == -1 || + ioapics[best].io_vector < ioapics[i].io_vector) + best = i; + } + if (best == -1) + return (ENOENT); + *apic = ioapics[best].io_apic; + *pin = intr - ioapics[best].io_vector; + if (*pin > 32) + printf("WARNING: Found intpin of %u for vector %d\n", *pin, + intr); + return (0); + } + + /* + * Parse an interrupt source override for an ISA interrupt. + */ + static void + madt_parse_interrupt_override(INTERRUPT_SOURCE_OVERRIDE *intr) + { + void *ioapic; + u_int pin; + + /* XXXTEST */ + printf("MADT: intr override: source %u, irq %u\n", intr->Source, + intr->GlobalSystemInterrupt); + KASSERT(intr->Bus == 0, ("bus for interrupt overrides must be zero")); + if (madt_find_interrupt(intr->GlobalSystemInterrupt, + &ioapic, &pin) != 0) { + printf("MADT: Could not find APIC for vector %d (IRQ %d)\n", + intr->GlobalSystemInterrupt, intr->Source); + return; + } + + if (intr->Source != intr->GlobalSystemInterrupt) + ioapic_remap_vector(ioapic, pin, intr->Source); + ioapic_set_triggermode(ioapic, pin, + interrupt_trigger(intr->TriggerMode)); + ioapic_set_polarity(ioapic, pin, interrupt_polarity(intr->Polarity)); + } + + /* + * Parse an entry for an NMI routed to an IO APIC. + */ + static void + madt_parse_nmi(NMI *nmi) + { + void *ioapic; + u_int pin; + + if (madt_find_interrupt(nmi->GlobalSystemInterrupt, + &ioapic, &pin) != 0) { + printf("MADT: Could not find APIC for vector %d\n", + nmi->GlobalSystemInterrupt); + return; + } + + ioapic_set_nmi(ioapic, pin); + if (nmi->TriggerMode != APIC_TRIGGER_CONFORM) + ioapic_set_triggermode(ioapic, pin, + interrupt_trigger(nmi->TriggerMode)); + if (nmi->Polarity != APIC_TRIGGER_CONFORM) + ioapic_set_polarity(ioapic, pin, + interrupt_polarity(nmi->Polarity)); + } + + /* + * Parse an entry for an NMI routed to a local APIC LVT pin. + */ + static void + madt_parse_local_nmi(LAPIC_NMI *nmi) + { + u_int apic_id, pin; + + if (nmi->ProcessorApicId == 0xff) + apic_id = APIC_ID_ALL; + else if (madt_find_cpu(nmi->ProcessorApicId, &apic_id) != 0) { + #if 0 + if (bootverbose) + #endif + printf("MADT: Ignoring local NMI routed to ACPI CPU %u\n", + nmi->ProcessorApicId); + return; + } + if (nmi->LINTPin == 0) + pin = LVT_LINT0; + else + pin = LVT_LINT1; + lapic_set_lvt_mode(apic_id, pin, APIC_LVT_DM_NMI); + if (nmi->TriggerMode != APIC_TRIGGER_CONFORM) + lapic_set_lvt_triggermode(apic_id, pin, + interrupt_trigger(nmi->TriggerMode)); + if (nmi->Polarity != APIC_POLARITY_CONFORM) + lapic_set_lvt_polarity(apic_id, pin, + interrupt_polarity(nmi->Polarity)); + } + + /* + * Parse interrupt entries. + */ + static void + madt_parse_ints(APIC_HEADER *entry, void *arg __unused) + { + + switch (entry->Type) { + case APIC_INTERRUPT_SOURCE_OVERRIDE: + madt_parse_interrupt_override( + (INTERRUPT_SOURCE_OVERRIDE *)entry); + break; + case APIC_NMI: + madt_parse_nmi((NMI *)entry); + break; + case APIC_LOCAL_APIC_NMI: + madt_parse_local_nmi((LAPIC_NMI *)entry); + break; + } + } Index: i386/conf/GENERIC =========================================================================== --- sys/i386/conf/GENERIC 2003/10/30 14:48:07 #79 +++ i386/conf/GENERIC 2003/10/30 14:48:07 @@ -66,8 +66,8 @@ options WITNESS_SKIPSPIN #Don't run witness on spinlocks for speed # To make an SMP kernel, the next two are needed -#options SMP # Symmetric MultiProcessor Kernel -#options APIC_IO # Symmetric (APIC) I/O +options SMP # Symmetric MultiProcessor Kernel +device apic # I/O APIC device isa device eisa Index: i386/conf/NOTES =========================================================================== --- sys/i386/conf/NOTES 2003/10/30 14:48:07 #157 +++ i386/conf/NOTES 2003/10/30 14:48:07 @@ -22,7 +22,7 @@ ##################################################################### # SMP OPTIONS: # -# APIC_IO enables the use of the IO APIC for Symmetric I/O. +# The apic device enables the use of the IO APIC for Symmetric I/O. # # Notes: # @@ -35,7 +35,7 @@ # # Mandatory: -options APIC_IO # Symmetric (APIC) I/O +device apic # I/O apic # # Rogue SMP hardware: Index: i386/conf/PAE =========================================================================== --- sys/i386/conf/PAE 2003/10/30 14:48:07 #7 +++ i386/conf/PAE 2003/10/30 14:48:07 @@ -10,10 +10,6 @@ # To make a PAE kernel, the next option is needed options PAE # Physical Address Extensions Kernel -# To make an SMP kernel, the next two are needed -#options SMP # Symmetric MultiProcessor Kernel -#options APIC_IO # Symmetric (APIC) I/O - # Compile acpi in statically since the module isn't built properly. Most # machines which support large amounts of memory require acpi. device acpi ==== //depot/vendor/freebsd/src/sys/i386/conf/SMP#4 - === Index: i386/i386/autoconf.c =========================================================================== --- sys/i386/i386/autoconf.c 2003/10/30 14:48:07 #18 +++ i386/i386/autoconf.c 2003/10/30 14:48:07 @@ -77,11 +77,6 @@ #include #include -#ifdef APIC_IO -#include -#else -#include -#endif /* APIC_IO */ #ifdef DEV_ISA #include @@ -114,28 +109,11 @@ { /* - * Activate the ICU's. Note that we are explicitly at splhigh() - * at present as we have no way to disable stray PCI level triggered - * interrupts until the devices have had a driver attached. This - * is particularly a problem when the interrupts are shared. For - * example, if IRQ 10 is shared between a disk and network device - * and the disk device generates an interrupt, if we "activate" - * IRQ 10 when the network driver is set up, then we will get - * recursive interrupt 10's as nothing will know how to turn off - * the disk device's interrupt. - * - * Having the ICU's active means we can probe interrupt routing to - * see if a device causes the corresponding pending bit to be set. - * - * This is all rather inconvenient. + * Enable interrupts on the processor. The interrupts are still + * disabled in the interrupt controllers until interrupt handlers + * are registered. */ -#ifdef APIC_IO - bsp_apic_configure(); - enable_intr(); -#else enable_intr(); - INTREN(IRQ_SLAVE); -#endif /* APIC_IO */ /* nexus0 is the top of the i386 device tree */ device_add_child(root_bus, "nexus", 0); @@ -151,12 +129,6 @@ if (isa_bus_device) isa_probe_children(isa_bus_device); #endif - - /* - * Now we're ready to handle (pending) interrupts. - * XXX this is slightly misplaced. - */ - spl0(); } static void @@ -167,11 +139,6 @@ cninit_finish(); if (bootverbose) { - -#ifdef APIC_IO - imen_dump(); -#endif /* APIC_IO */ - #ifdef PC98 { int i; Index: i386/i386/bios.c =========================================================================== --- sys/i386/i386/bios.c 2003/10/30 14:48:07 #21 +++ i386/i386/bios.c 2003/10/30 14:48:07 @@ -583,20 +583,14 @@ } /* - * If we are in APIC_IO mode, we should ignore the ISA PIC if it - * shows up. Likewise, in !APIC_IO mode, we should ignore the - * APIC (less important). - * This is significant because the ISA PIC will claim IRQ 2 (which - * it uses for chaining), while in APIC mode this is a valid IRQ - * available for general use. + * Ignore PICs so that we don't have to worry about the PICs + * claiming IRQs to prevent their use. The PIC drivers + * already ensure that invalid IRQs are not used. */ -#ifdef APIC_IO if (!strcmp(pnp_eisaformat(pd->devid), "PNP0000")) /* ISA PIC */ continue; -#else if (!strcmp(pnp_eisaformat(pd->devid), "PNP0003")) /* APIC */ continue; -#endif /* Add the device and parse its resources */ dev = BUS_ADD_CHILD(parent, ISA_ORDER_PNP, NULL, -1); Index: i386/i386/critical.c =========================================================================== --- sys/i386/i386/critical.c 2003/10/30 14:48:07 #11 +++ i386/i386/critical.c 2003/10/30 14:48:07 @@ -30,144 +30,17 @@ #include #include -#include -#include -#include -#include -#include #include -#include -#include -#include #include -#ifdef SMP -#include -#include -#else -/* - * XXX this mess to get sched_ithd() and call_fast_unpend() - */ -#include -#include -#include -#include -#include -#endif - -void i386_unpend(void); /* NOTE: not static, called from assembly */ - /* - * cpu_unpend() - called from critical_exit() inline after quick - * interrupt-pending check. - */ -void -cpu_unpend(void) -{ - register_t eflags; - struct thread *td; - - td = curthread; - eflags = intr_disable(); - if (PCPU_GET(int_pending)) { - ++td->td_intr_nesting_level; - i386_unpend(); - --td->td_intr_nesting_level; - } - intr_restore(eflags); -} - -/* * cpu_critical_fork_exit() - cleanup after fork + * + * Enable interrupts in the saved copy of eflags. */ void cpu_critical_fork_exit(void) { - enable_intr(); -} - -/* - * Called from cpu_unpend or called from the assembly vector code - * to process any interrupts which may have occured while we were in - * a critical section. - * - * - interrupts must be disabled - * - td_critnest must be 0 - * - td_intr_nesting_level must be incremented by the caller - * - * NOT STATIC (called from assembly) - */ -void -i386_unpend(void) -{ - struct clockframe frame; - - frame.cf_cs = SEL_KPL; - frame.cf_eip = (register_t)i386_unpend; - frame.cf_eflags = PSL_KERNEL; - KASSERT(curthread->td_critnest == 0, ("unpend critnest != 0")); - KASSERT((read_eflags() & PSL_I) == 0, ("unpend interrupts enabled1")); - curthread->td_critnest = 1; - for (;;) { - u_int32_t mask; - int irq; - - /* - * Fast interrupts have priority - */ - if ((mask = PCPU_GET(fpending)) != 0) { - irq = bsfl(mask); - PCPU_SET(fpending, mask & ~(1 << irq)); - call_fast_unpend(irq); - KASSERT((read_eflags() & PSL_I) == 0, - ("unpend interrupts enabled2 %d", irq)); - continue; - } - - /* - * Threaded interrupts come next - */ - if ((mask = PCPU_GET(ipending)) != 0) { - irq = bsfl(mask); - PCPU_SET(ipending, mask & ~(1 << irq)); - sched_ithd((void *)irq); - KASSERT((read_eflags() & PSL_I) == 0, - ("unpend interrupts enabled3 %d", irq)); - continue; - } - - /* - * Software interrupts and delayed IPIs are last - * - * XXX give the bits #defined names. see also - * isa/xxx_vector.s - */ - if ((mask = PCPU_GET(spending)) != 0) { - irq = bsfl(mask); - PCPU_SET(spending, mask & ~(1 << irq)); - switch(irq) { - case 0: /* bit 0 - hardclock */ - hardclock_process(&frame); - break; - case 1: /* bit 1 - statclock */ - if (profprocs != 0) - profclock(&frame); - if (pscnt == psdiv) - statclock(&frame); - break; - } - KASSERT((read_eflags() & PSL_I) == 0, - ("unpend interrupts enabled4 %d", irq)); - continue; - } - break; - } - /* - * Interrupts are still disabled, we can safely clear int_pending - * and td_critnest. - */ - KASSERT((read_eflags() & PSL_I) == 0, ("unpend interrupts enabled5")); - PCPU_SET(int_pending, 0); - curthread->td_critnest = 0; + curthread->td_md.md_savecrit = read_eflags() | PSL_I; } Index: i386/i386/db_interface.c =========================================================================== --- sys/i386/i386/db_interface.c 2003/10/30 14:48:07 #16 +++ i386/i386/db_interface.c 2003/10/30 14:48:07 @@ -334,5 +334,6 @@ db_show_mdpcpu(struct pcpu *pc) { + db_printf("APIC ID = %d\n", pc->pc_apic_id); db_printf("currentldt = 0x%x\n", pc->pc_currentldt); } Index: i386/i386/db_trace.c =========================================================================== --- sys/i386/i386/db_trace.c 2003/10/30 14:48:07 #15 +++ i386/i386/db_trace.c 2003/10/30 14:48:07 @@ -225,8 +225,8 @@ if (strcmp(name, "calltrap") == 0 || strcmp(name, "fork_trampoline") == 0) frame_type = TRAP; - else if (strncmp(name, "Xintr", 5) == 0 || - strncmp(name, "Xfastintr", 9) == 0) + else if (strncmp(name, "Xatpic_intr", 11) == 0 || + strncmp(name, "Xapic_isr", 9) == 0) frame_type = INTERRUPT; else if (strcmp(name, "Xlcall_syscall") == 0 || strcmp(name, "Xint0x80_syscall") == 0) Index: i386/i386/exception.s =========================================================================== --- sys/i386/i386/exception.s 2003/10/30 14:48:07 #18 +++ i386/i386/exception.s 2003/10/30 14:48:07 @@ -37,12 +37,8 @@ #include "opt_npx.h" #include -#include #include #include -#ifdef SMP -#include /** various SMP options */ -#endif #include "assym.s" @@ -77,9 +73,6 @@ * %ss segment registers, but does not mess with %ds, %es, or %fs. Thus we * must load them with appropriate values for supervisor mode operation. */ -#define IDTVEC(name) ALIGN_TEXT; .globl __CONCAT(X,name); \ - .type __CONCAT(X,name),@function; __CONCAT(X,name): -#define TRAP(a) pushl $(a) ; jmp alltraps MCOUNT_LABEL(user) MCOUNT_LABEL(btrap) @@ -233,13 +226,6 @@ */ #include "i386/i386/vm86bios.s" -/* - * Include what was once config+isa-dependent code. - * XXX it should be in a stand-alone file. It's still icu-dependent and - * belongs in i386/isa. - */ -#include "i386/isa/vector.s" - .data ALIGN_DATA @@ -250,6 +236,7 @@ */ .text SUPERALIGN_TEXT + .globl doreti .type doreti,@function doreti: FAKE_MCOUNT(bintr) /* init "from" bintr -> doreti */ @@ -334,9 +321,3 @@ movl $0,TF_ERR(%esp) /* XXX should be the error code */ movl $T_PROTFLT,TF_TRAPNO(%esp) jmp alltraps_with_regs_pushed - -#ifdef APIC_IO -#include "i386/isa/apic_ipl.s" -#else -#include "i386/isa/icu_ipl.s" -#endif /* APIC_IO */ Index: i386/i386/genassym.c =========================================================================== --- sys/i386/i386/genassym.c 2003/10/30 14:48:07 #33 +++ i386/i386/genassym.c 2003/10/30 14:48:07 @@ -39,6 +39,7 @@ #include __FBSDID("$FreeBSD: src/sys/i386/i386/genassym.c,v 1.144 2003/10/01 23:46:07 peter Exp $"); +#include "opt_apic.h" #include "opt_compat.h" #include "opt_kstack_pages.h" @@ -70,8 +71,8 @@ #include #include #include -#ifdef SMP -#include +#ifdef DEV_APIC +#include #endif #include #include @@ -178,6 +179,7 @@ ASSYM(ENOENT, ENOENT); ASSYM(EFAULT, EFAULT); ASSYM(ENAMETOOLONG, ENAMETOOLONG); +ASSYM(MAXCOMLEN, MAXCOMLEN); ASSYM(MAXPATHLEN, MAXPATHLEN); ASSYM(BOOTINFO_SIZE, sizeof(struct bootinfo)); ASSYM(BI_VERSION, offsetof(struct bootinfo, bi_version)); @@ -192,10 +194,6 @@ ASSYM(PC_SIZEOF, sizeof(struct pcpu)); ASSYM(PC_PRVSPACE, offsetof(struct pcpu, pc_prvspace)); ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread)); -ASSYM(PC_INT_PENDING, offsetof(struct pcpu, pc_int_pending)); -ASSYM(PC_IPENDING, offsetof(struct pcpu, pc_ipending)); -ASSYM(PC_FPENDING, offsetof(struct pcpu, pc_fpending)); -ASSYM(PC_SPENDING, offsetof(struct pcpu, pc_spending)); ASSYM(PC_FPCURTHREAD, offsetof(struct pcpu, pc_fpcurthread)); ASSYM(PC_IDLETHREAD, offsetof(struct pcpu, pc_idlethread)); ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb)); @@ -206,13 +204,14 @@ ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid)); ASSYM(PC_CURPMAP, offsetof(struct pcpu, pc_curpmap)); -#ifdef SMP +#ifdef DEV_APIC ASSYM(LA_VER, offsetof(struct LAPIC, version)); ASSYM(LA_TPR, offsetof(struct LAPIC, tpr)); ASSYM(LA_EOI, offsetof(struct LAPIC, eoi)); ASSYM(LA_SVR, offsetof(struct LAPIC, svr)); ASSYM(LA_ICR_LO, offsetof(struct LAPIC, icr_lo)); ASSYM(LA_ICR_HI, offsetof(struct LAPIC, icr_hi)); +ASSYM(LA_ISR, offsetof(struct LAPIC, isr0)); #endif ASSYM(KCSEL, GSEL(GCODE_SEL, SEL_KPL)); Index: i386/i386/identcpu.c =========================================================================== --- sys/i386/i386/identcpu.c 2003/10/30 14:48:07 #38 +++ i386/i386/identcpu.c 2003/10/30 14:48:07 @@ -55,12 +55,10 @@ #include #include #include +#include +#include #include #include -#include - -#include -#include #define IDENTBLUE_CYRIX486 0 #define IDENTBLUE_IBMCPU 1 Index: i386/i386/initcpu.c =========================================================================== --- sys/i386/i386/initcpu.c 2003/10/30 14:48:07 #19 +++ i386/i386/initcpu.c 2003/10/30 14:48:07 @@ -474,7 +474,6 @@ static void init_ppro(void) { -#ifndef SMP u_int64_t apicbase; /* @@ -483,7 +482,6 @@ apicbase = rdmsr(MSR_APICBASE); apicbase &= ~APICBASE_ENABLED; wrmsr(MSR_APICBASE, apicbase); -#endif } /* Index: i386/i386/intr_machdep.c =========================================================================== *** /dev/null Thu Oct 30 14:47:44 2003 --- i386/i386/intr_machdep.c Thu Oct 30 14:48:09 2003 *************** *** 0 **** --- 1,296 ---- + /*- + * Copyright (c) 2003 John Baldwin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the author nor the names of any co-contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + /* + * Machine dependent interrupt code for i386. For the i386, we have to + * deal with different PICs. Thus, we use the passed in vector to lookup + * an interrupt source associated with that vector. The interrupt source + * describes which PIC the source belongs to and includes methods to handle + * that source. + */ + + #include "opt_ddb.h" + + #include + #include + #include + #include + #include + #include + #include + #include + #include + #include + #include + #ifdef DDB + #include + #endif + + #define MAX_STRAY_LOG 5 + + typedef void (*mask_fn)(int vector); + + static int intrcnt_index; + static struct intsrc *interrupt_sources[NUM_IO_INTS]; + static struct mtx intr_table_lock; + + static void intr_init(void *__dummy); + static void intrcnt_setname(const char *name, int index); + static void intrcnt_updatename(struct intsrc *is); + static void intrcnt_register(struct intsrc *is); + + /* + * Register a new interrupt source with the global interrupt system. + * The global interrupts need to be disabled when this function is + * called. + */ + int + intr_register_source(struct intsrc *isrc) + { + int error, vector; + + vector = isrc->is_pic->pic_vector(isrc); + if (interrupt_sources[vector] != NULL) + return (EEXIST); + /* + * Ok, so this is kind of a nasty optimization that only works + * because sizeof(int) == sizeof(void *) on i386. If we passed + * in the actual vector to ithread_create and then used wrapper + * functions for disable_intsrc and enable_intsrc, then we'd + * have to go lookup in the table everytime we enabled/disabled + * the interrupt source. That involves looking at a lock, etc. + * and is just ugly. Instead, we cast the pointer to the intsrc + * to an int (yuck) and pass in the actual PIC methods meaning + * that when we enable/disable an interrupt we call the PIC + * methods directly. + */ + error = ithread_create(&isrc->is_ithread, (intptr_t)isrc, 0, + (mask_fn)isrc->is_pic->pic_disable_source, + (mask_fn)isrc->is_pic->pic_enable_source, "irq%d:", vector); + if (error) + return (error); + mtx_lock_spin(&intr_table_lock); + if (interrupt_sources[vector] != NULL) { + mtx_unlock_spin(&intr_table_lock); + ithread_destroy(isrc->is_ithread); + return (EEXIST); + } + intrcnt_register(isrc); + interrupt_sources[vector] = isrc; + mtx_unlock_spin(&intr_table_lock); + return (0); + } + + struct intsrc * + intr_lookup_source(int vector) + { + + return (interrupt_sources[vector]); + } + + int + intr_add_handler(const char *name, int vector, driver_intr_t handler, + void *arg, enum intr_type flags, void **cookiep) + { + struct intsrc *isrc; + int error; + + isrc = intr_lookup_source(vector); + if (isrc == NULL) + return (EINVAL); + error = ithread_add_handler(isrc->is_ithread, name, handler, arg, + ithread_priority(flags), flags, cookiep); + if (error == 0) { + intrcnt_updatename(isrc); + isrc->is_pic->pic_enable_intr(isrc); + isrc->is_pic->pic_enable_source(isrc); + } + return (error); + } + + int + intr_remove_handler(void *cookie) + { + int error; + + error = ithread_remove_handler(cookie); + #ifdef XXX + if (error == 0) + intrcnt_updatename(/* XXX */); + #endif + return (error); + } + + void + intr_execute_handlers(struct intsrc *isrc, struct intrframe *iframe) + { + struct ithd *it; + struct intrhand *ih; + int error, vector; + + /* + * We count software interrupts when we process them. The + * code here follows previous practice, but there's an + * argument for counting hardware interrupts when they're + * processed too. + */ + atomic_add_long(isrc->is_count, 1); + atomic_add_int(&cnt.v_intr, 1); + + /* + * Execute fast interrupt handlers directly. + * To support clock handlers, if a handler registers + * with a NULL argument, then we pass it a pointer to + * a trapframe as its argument. + */ + it = isrc->is_ithread; + ih = TAILQ_FIRST(&it->it_handlers); + critical_enter(); + if (ih == NULL) + error = EINVAL; + else if (ih->ih_flags & IH_FAST) { + TAILQ_FOREACH(ih, &it->it_handlers, ih_next) { + MPASS(ih->ih_flags & IH_FAST); + CTR3(KTR_INTR, "%s: executing handler %p(%p)", + __func__, ih->ih_handler, + ih->ih_argument == NULL ? iframe : + ih->ih_argument); + if (ih->ih_argument == NULL) + ih->ih_handler(iframe); + else + ih->ih_handler(ih->ih_argument); + } + isrc->is_pic->pic_enable_source(isrc); + error = 0; + } else + error = ithread_schedule(it, !cold); + critical_exit(); + if (error == EINVAL) { + vector = isrc->is_pic->pic_vector(isrc); + atomic_add_long(isrc->is_straycount, 1); + if (*isrc->is_straycount < MAX_STRAY_LOG) + log(LOG_ERR, "stray irq%d\n", vector); + else if (*isrc->is_straycount == MAX_STRAY_LOG) + log(LOG_CRIT, + "too many stray irq %d's: not logging anymore\n", + vector); + } + } + + void + intr_resume(void) + { + struct intsrc **isrc; + int i; + + mtx_lock_spin(&intr_table_lock); + for (i = 0, isrc = interrupt_sources; i < NUM_IO_INTS; i++, isrc++) + if (*isrc != NULL && (*isrc)->is_pic->pic_resume != NULL) + (*isrc)->is_pic->pic_resume(*isrc); + mtx_unlock_spin(&intr_table_lock); + } + + void + intr_suspend(void) + { + struct intsrc **isrc; + int i; + + mtx_lock_spin(&intr_table_lock); + for (i = 0, isrc = interrupt_sources; i < NUM_IO_INTS; i++, isrc++) + if (*isrc != NULL && (*isrc)->is_pic->pic_suspend != NULL) + (*isrc)->is_pic->pic_suspend(*isrc); + mtx_unlock_spin(&intr_table_lock); + } + + static void + intrcnt_setname(const char *name, int index) + { + + snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s", + MAXCOMLEN, name); + } + + static void + intrcnt_updatename(struct intsrc *is) + { + + intrcnt_setname(is->is_ithread->it_td->td_proc->p_comm, is->is_index); + } + + static void + intrcnt_register(struct intsrc *is) + { + char straystr[MAXCOMLEN + 1]; + + /* mtx_assert(&intr_table_lock, MA_OWNED); */ + KASSERT(is->is_ithread != NULL, ("%s: isrc with no ithread", __func__)); + is->is_index = intrcnt_index; + intrcnt_index += 2; + snprintf(straystr, MAXCOMLEN + 1, "stray irq%d", + is->is_pic->pic_vector(is)); + intrcnt_updatename(is); + is->is_count = &intrcnt[is->is_index]; + intrcnt_setname(straystr, is->is_index + 1); + is->is_straycount = &intrcnt[is->is_index + 1]; + } + + static void + intr_init(void *dummy __unused) + { + + intrcnt_setname("???", 0); + intrcnt_index = 1; + mtx_init(&intr_table_lock, "intr table", NULL, MTX_SPIN); + } + SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL) + + #ifdef DDB + /* + * Dump data about interrupt handlers + */ + DB_SHOW_COMMAND(irqs, db_show_irqs) + { + struct intsrc **isrc; + int i, quit, verbose; + + quit = 0; + if (strcmp(modif, "v") == 0) + verbose = 1; + else + verbose = 0; + isrc = interrupt_sources; + db_setup_paging(db_simple_pager, &quit, DB_LINES_PER_PAGE); + for (i = 0; i < NUM_IO_INTS && !quit; i++, isrc++) + if (*isrc != NULL) + db_dump_ithread((*isrc)->is_ithread, verbose); + } + #endif Index: i386/i386/io_apic.c =========================================================================== *** /dev/null Thu Oct 30 14:47:44 2003 --- i386/i386/io_apic.c Thu Oct 30 14:48:09 2003 *************** *** 0 **** --- 1,695 ---- + /*- + * Copyright (c) 2003 John Baldwin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the author nor the names of any co-contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + #include "opt_isa.h" + #include "opt_no_mixed_mode.h" + + #include + #include + #include + #include + #include + #include + #include + + #include + #include + + #include + #include + #include + #include + #include + + #if defined(DEV_ISA) && !defined(NO_MIXED_MODE) + #define MIXED_MODE + #endif + + #define IOAPIC_ISA_INTS 16 + #define IOAPIC_MEM_REGION 32 + #define IOAPIC_REDTBL_LO(i) (IOAPIC_REDTBL + (i) * 2) + #define IOAPIC_REDTBL_HI(i) (IOAPIC_REDTBL_LO(i) + 1) + + #define VECTOR_EXTINT -1 + #define VECTOR_NMI -2 + #define VECTOR_SMI -3 + #define VECTOR_DISABLED -4 + + #define DEST_NONE -1 + #define DEST_EXTINT -2 + + #define TODO printf("%s: not implemented!\n", __func__) + + MALLOC_DEFINE(M_IOAPIC, "I/O APIC", "I/O APIC structures"); + + /* + * New interrupt support code.. + * + * XXX: we really should have the interrupt cookie passed up from new-bus + * just be a int pin, and not map 1:1 to interrupt vector number but should + * use INTR_TYPE_FOO to set priority bands for device classes and do all the + * magic remapping of intpin to vector in here. For now we just cheat as on + * ia64 and map intpin X to vector NRSVIDT + X. Note that we assume that the + * first IO APIC has ISA interrupts on pins 1-15. Not sure how you are + * really supposed to figure out which IO APIC in a system with multiple IO + * APIC's actually has the ISA interrupts routed to it. As far as interrupt + * pin numbers, we use the ACPI System Interrupt number model where each + * IO APIC has a contiguous chunk of the System Interrupt address space. + */ + + struct ioapic_intsrc { + struct intsrc io_intsrc; + int io_intpin:8; + int io_vector:8; + int io_activehi:1; + int io_edgetrigger:1; + int io_masked:1; + int io_dest:5; + }; + + struct ioapic { + struct pic io_pic; + u_int io_id:8; /* logical ID */ + u_int io_apic_id:4; + u_int io_intbase:8; /* System Interrupt base */ + u_int io_numintr:8; + volatile ioapic_t *io_addr; /* XXX: should use bus_space */ + STAILQ_ENTRY(ioapic) io_next; + struct ioapic_intsrc io_pins[0]; + }; + + static STAILQ_HEAD(,ioapic) ioapic_list = STAILQ_HEAD_INITIALIZER(ioapic_list); + static u_int next_id, program_logical_dest; + + static u_int ioapic_read(volatile ioapic_t *apic, int reg); + static void ioapic_write(volatile ioapic_t *apic, int reg, u_int val); + static void ioapic_enable_source(struct intsrc *isrc); + static void ioapic_disable_source(struct intsrc *isrc); + static void ioapic_eoi_source(struct intsrc *isrc); + static void ioapic_enable_intr(struct intsrc *isrc); + static int ioapic_vector(struct intsrc *isrc); + static int ioapic_source_pending(struct intsrc *isrc); + static void ioapic_suspend(struct intsrc *isrc); + static void ioapic_resume(struct intsrc *isrc); + static void ioapic_program_destination(struct ioapic_intsrc *intpin); + #ifdef MIXED_MODE + static void ioapic_setup_mixed_mode(struct ioapic_intsrc *intpin); + #endif + + struct pic ioapic_template = { ioapic_enable_source, ioapic_disable_source, + ioapic_eoi_source, ioapic_enable_intr, + ioapic_vector, ioapic_source_pending, + ioapic_suspend, ioapic_resume }; + + static int next_ioapic_base, logical_clusters, current_cluster; + + static u_int + ioapic_read(volatile ioapic_t *apic, int reg) + { + + mtx_assert(&icu_lock, MA_OWNED); + apic->ioregsel = reg; + return (apic->iowin); + } + + static void + ioapic_write(volatile ioapic_t *apic, int reg, u_int val) + { + + mtx_assert(&icu_lock, MA_OWNED); + apic->ioregsel = reg; + apic->iowin = val; + } + + static void + ioapic_enable_source(struct intsrc *isrc) + { + struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; + struct ioapic *io = (struct ioapic *)isrc->is_pic; + uint32_t flags; + + mtx_lock_spin(&icu_lock); + if (intpin->io_masked) { + flags = ioapic_read(io->io_addr, + IOAPIC_REDTBL_LO(intpin->io_intpin)); + flags &= ~(IOART_INTMASK); + ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), + flags); + intpin->io_masked = 0; + } + mtx_unlock_spin(&icu_lock); + } + + static void + ioapic_disable_source(struct intsrc *isrc) + { + struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; + struct ioapic *io = (struct ioapic *)isrc->is_pic; + uint32_t flags; + + mtx_lock_spin(&icu_lock); + if (!intpin->io_masked && !intpin->io_edgetrigger) { + flags = ioapic_read(io->io_addr, + IOAPIC_REDTBL_LO(intpin->io_intpin)); + flags |= IOART_INTMSET; + ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), + flags); + intpin->io_masked = 1; + } + mtx_unlock_spin(&icu_lock); + } + + static void + ioapic_eoi_source(struct intsrc *isrc) + { + TODO; + /* lapic_eoi(); */ + } + + /* + * Program an individual intpin's logical destination. + */ + static void + ioapic_program_destination(struct ioapic_intsrc *intpin) + { + struct ioapic *io = (struct ioapic *)intpin->io_intsrc.is_pic; + uint32_t value; + + KASSERT(intpin->io_dest != DEST_NONE, + ("intpin not assigned to a cluster")); + KASSERT(intpin->io_dest != DEST_EXTINT, + ("intpin routed via ExtINT")); + /* XXXTEST */ + printf("ioapic%u: routing intpin %u (", io->io_id, intpin->io_intpin); + if (intpin->io_vector == VECTOR_EXTINT) + printf("ExtINT"); + else + printf("IRQ %u", intpin->io_vector); + printf(") to cluster %u\n", intpin->io_dest); + mtx_lock_spin(&icu_lock); + value = ioapic_read(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin)); + value &= ~IOART_DESTMOD; + value |= IOART_DESTLOG; + ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), value); + value = ioapic_read(io->io_addr, IOAPIC_REDTBL_HI(intpin->io_intpin)); + value &= ~IOART_DEST; + value |= (intpin->io_dest << APIC_ID_CLUSTER_SHIFT | + APIC_ID_CLUSTER_ID) << APIC_ID_SHIFT; + ioapic_write(io->io_addr, IOAPIC_REDTBL_HI(intpin->io_intpin), value); + mtx_unlock_spin(&icu_lock); + } + + static void + ioapic_assign_cluster(struct ioapic_intsrc *intpin) + { + /* + * Assign this intpin to a logical APIC cluster in a + * round-robin fashion. We don't actually use the logical + * destination for this intpin until after all the CPU's + * have been started so that we don't end up with interrupts + * that don't go anywhere. Another alternative might be to + * start up the CPU's earlier so that they can handle interrupts + * sooner. + */ + intpin->io_dest = current_cluster; + current_cluster++; + if (current_cluster >= logical_clusters) + current_cluster = 0; + if (program_logical_dest) + ioapic_program_destination(intpin); + } + + static void + ioapic_enable_intr(struct intsrc *isrc) + { + struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; + + KASSERT(intpin->io_dest != DEST_EXTINT, + ("ExtINT pin trying to use ioapic enable_intr method")); + if (intpin->io_dest == DEST_NONE) { + ioapic_assign_cluster(intpin); + lapic_enable_intr(intpin->io_vector); + } + } + + static int + ioapic_vector(struct intsrc *isrc) + { + struct ioapic_intsrc *pin; + + pin = (struct ioapic_intsrc *)isrc; + return (pin->io_vector); + } + + static int + ioapic_source_pending(struct intsrc *isrc) + { + struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; + + return (lapic_intr_pending(intpin->io_vector)); + } + + static void + ioapic_suspend(struct intsrc *isrc) + { + + TODO; + } + + static void + ioapic_resume(struct intsrc *isrc) + { + + TODO; + } + + /* + * Allocate and return a logical cluster ID. Note that the first time + * this is called, it returns cluster 0. ioapic_enable_intr() treats + * the two cases of logical_clusters == 0 and logical_clusters == 1 the + * same: one cluster of ID 0 exists. The logical_clusters == 0 case is + * for UP kernels, which should never call this function. + */ + int + ioapic_next_logical_cluster(void) + { + + if (logical_clusters >= APIC_MAX_CLUSTER) + panic("WARNING: Local APIC cluster IDs exhausted!"); + return (logical_clusters++); + } + + /* + * Create a plain I/O APIC object. + */ + void * + ioapic_create(uintptr_t addr, int32_t apic_id, int intbase) + { + struct ioapic *io; + struct ioapic_intsrc *intpin; + volatile ioapic_t *apic; + uintptr_t poffs; + u_int numintr, i; + uint32_t value; + + poffs = addr - trunc_page(addr); + apic = (ioapic_t *)((char *)pmap_mapdev(addr - poffs, + poffs + IOAPIC_MEM_REGION) + poffs); + mtx_lock_spin(&icu_lock); + numintr = ((ioapic_read(apic, IOAPIC_VER) & IOART_VER_MAXREDIR) >> + MAXREDIRSHIFT) + 1; + mtx_unlock_spin(&icu_lock); + io = malloc(sizeof(struct ioapic) + + numintr * sizeof(struct ioapic_intsrc), M_IOAPIC, M_WAITOK); + io->io_pic = ioapic_template; + mtx_lock_spin(&icu_lock); + io->io_id = next_id++; + io->io_apic_id = ioapic_read(apic, IOAPIC_ID) >> APIC_ID_SHIFT; + if (apic_id != -1 && io->io_apic_id != apic_id) { + ioapic_write(apic, IOAPIC_ID, apic_id << APIC_ID_SHIFT); + mtx_unlock_spin(&icu_lock); + io->io_apic_id = apic_id; + printf("ioapic%u: Changing APIC ID to %d\n", io->io_id, + apic_id); + } else + mtx_unlock_spin(&icu_lock); + if (intbase == -1) { + intbase = next_ioapic_base; + printf("ioapic%u: Assuming intbase of %d\n", io->io_id, + intbase); + } else if (intbase != next_ioapic_base) + printf("ioapic%u: WARNING: intbase %d != expected base %d\n", + io->io_id, intbase, next_ioapic_base); + io->io_intbase = intbase; + next_ioapic_base += numintr; + io->io_numintr = numintr; + io->io_addr = apic; + + /* + * Initialize pins. Start off with interrupts disabled. Default + * to active-hi and edge-triggered for ISA interrupts and active-lo + * and level-triggered for all others. + */ + bzero(io->io_pins, sizeof(struct ioapic_intsrc) * numintr); + mtx_lock_spin(&icu_lock); + for (i = 0, intpin = io->io_pins; i < numintr; i++, intpin++) { + intpin->io_intsrc.is_pic = (struct pic *)io; + intpin->io_intpin = i; + intpin->io_vector = intbase + i; + + /* + * Assume that pin 0 on the first IO APIC is an ExtINT pin by + * default. Assume that intpins 1-15 are ISA interrupts and + * use suitable defaults for those. Assume that all other + * intpins are PCI interrupts. Enable the ExtINT pin by + * default but mask all other pins. + */ + if (intpin->io_vector == 0) { + intpin->io_activehi = 1; + intpin->io_edgetrigger = 1; + intpin->io_vector = VECTOR_EXTINT; + intpin->io_masked = 0; + } else if (intpin->io_vector < IOAPIC_ISA_INTS) { + intpin->io_activehi = 1; + intpin->io_edgetrigger = 1; + intpin->io_masked = 1; + } else { + intpin->io_activehi = 0; + intpin->io_edgetrigger = 0; + intpin->io_masked = 1; + } + + /* + * Start off without a logical cluster destination until + * the pin is enabled. + */ + intpin->io_dest = DEST_NONE; + if (bootverbose) { + printf("ioapic%u: intpin %d -> ", io->io_id, i); + if (intpin->io_vector == VECTOR_EXTINT) + printf("ExtINT\n"); + else + printf("irq %d\n", intpin->io_vector); + } + value = ioapic_read(apic, IOAPIC_REDTBL_LO(i)); + ioapic_write(apic, IOAPIC_REDTBL_LO(i), value | IOART_INTMSET); + } + mtx_unlock_spin(&icu_lock); + + return (io); + } + + int + ioapic_get_vector(void *cookie, u_int pin) + { + struct ioapic *io; + + io = (struct ioapic *)cookie; + if (pin >= io->io_numintr) + return (-1); + return (io->io_pins[pin].io_vector); + } + + int + ioapic_disable_pin(void *cookie, u_int pin) + { + struct ioapic *io; + + io = (struct ioapic *)cookie; + if (pin >= io->io_numintr) + return (EINVAL); + if (io->io_pins[pin].io_vector == VECTOR_DISABLED) + return (EINVAL); + io->io_pins[pin].io_vector = VECTOR_DISABLED; + #if 0 + if (bootverbose) + #endif + printf("ioapic%u: intpin %d disabled\n", io->io_id, pin); + return (0); + } + + int + ioapic_remap_vector(void *cookie, u_int pin, int vector) + { + struct ioapic *io; + + io = (struct ioapic *)cookie; + if (pin >= io->io_numintr || vector < 0) + return (EINVAL); + if (io->io_pins[pin].io_vector < 0) + return (EINVAL); + io->io_pins[pin].io_vector = vector; + #if 0 + if (bootverbose) + #endif + printf("ioapic%u: Routing IRQ %d -> intpin %d\n", io->io_id, + vector, pin); + return (0); + } + + int + ioapic_set_nmi(void *cookie, u_int pin) + { + struct ioapic *io; + + io = (struct ioapic *)cookie; + if (pin >= io->io_numintr) + return (EINVAL); + if (io->io_pins[pin].io_vector < 0) + return (EINVAL); + io->io_pins[pin].io_vector = VECTOR_NMI; + io->io_pins[pin].io_masked = 0; + io->io_pins[pin].io_edgetrigger = 1; + io->io_pins[pin].io_activehi = 1; + #if 0 + if (bootverbose) + #endif + printf("ioapic%u: Routing NMI -> intpin %d\n", + io->io_id, pin); + return (0); + } + + int + ioapic_set_smi(void *cookie, u_int pin) + { + struct ioapic *io; + + io = (struct ioapic *)cookie; + if (pin >= io->io_numintr) + return (EINVAL); + if (io->io_pins[pin].io_vector < 0) + return (EINVAL); + io->io_pins[pin].io_vector = VECTOR_SMI; + io->io_pins[pin].io_masked = 0; + io->io_pins[pin].io_edgetrigger = 1; + io->io_pins[pin].io_activehi = 1; + #if 0 + if (bootverbose) + #endif + printf("ioapic%u: Routing SMI -> intpin %d\n", + io->io_id, pin); + return (0); + } + + int + ioapic_set_extint(void *cookie, u_int pin) + { + struct ioapic *io; + + io = (struct ioapic *)cookie; + if (pin >= io->io_numintr) + return (EINVAL); + if (io->io_pins[pin].io_vector < 0) + return (EINVAL); + io->io_pins[pin].io_vector = VECTOR_EXTINT; + io->io_pins[pin].io_masked = 0; + io->io_pins[pin].io_edgetrigger = 1; + io->io_pins[pin].io_activehi = 1; + #if 0 + if (bootverbose) + #endif + printf("ioapic%u: Routing external 8259A's -> intpin %d\n", + io->io_id, pin); + return (0); + } + + int + ioapic_set_polarity(void *cookie, u_int pin, char activehi) + { + struct ioapic *io; + + io = (struct ioapic *)cookie; + if (pin >= io->io_numintr) + return (EINVAL); + if (io->io_pins[pin].io_vector < 0) + return (EINVAL); + io->io_pins[pin].io_activehi = activehi; + if (bootverbose) + printf("ioapic%u: intpin %d polarity: %s\n", io->io_id, pin, + activehi ? "active-hi" : "active-lo"); + return (0); + } + + int + ioapic_set_triggermode(void *cookie, u_int pin, char edgetrigger) + { + struct ioapic *io; + + io = (struct ioapic *)cookie; + if (pin >= io->io_numintr) + return (EINVAL); + if (io->io_pins[pin].io_vector < 0) + return (EINVAL); + io->io_pins[pin].io_edgetrigger = edgetrigger; + if (bootverbose) + printf("ioapic%u: intpin %d trigger: %s\n", io->io_id, pin, + edgetrigger ? "edge" : "level"); + return (0); + } + + /* + * Register a complete I/O APIC object with the interrupt subsystem. + */ + void + ioapic_register(void *cookie) + { + struct ioapic_intsrc *pin; + struct ioapic *io; + volatile ioapic_t *apic; + uint32_t flags; + int i; + + io = (struct ioapic *)cookie; + apic = io->io_addr; + mtx_lock_spin(&icu_lock); + flags = ioapic_read(apic, IOAPIC_VER) & IOART_VER_VERSION; + STAILQ_INSERT_TAIL(&ioapic_list, io, io_next); + mtx_unlock_spin(&icu_lock); + printf("ioapic%u irqs %u-%u on motherboard\n", io->io_id, + flags, io->io_intbase, io->io_intbase + io->io_numintr - 1); + for (i = 0, pin = io->io_pins; i < io->io_numintr; i++, pin++) { + /* + * Finish initializing the pins by programming the vectors + * and delivery mode. XXX this may not be all right yet + */ + if (pin->io_vector == VECTOR_DISABLED) + continue; + flags = IOART_DESTPHY; + if (pin->io_edgetrigger) + flags |= IOART_TRGREDG; + else + flags |= IOART_TRGRLVL; + if (pin->io_activehi) + flags |= IOART_INTAHI; + else + flags |= IOART_INTALO; + if (pin->io_masked) + flags |= IOART_INTMSET; + switch (pin->io_vector) { + case VECTOR_EXTINT: + KASSERT(pin->io_edgetrigger, + ("EXTINT not edge triggered")); + flags |= IOART_DELEXINT; + break; + case VECTOR_NMI: + KASSERT(pin->io_edgetrigger, + ("NMI not edge triggered")); + flags |= IOART_DELNMI; + break; + case VECTOR_SMI: + KASSERT(pin->io_edgetrigger, + ("SMI not edge triggered")); + flags |= IOART_DELSMI; + break; + default: + flags |= IOART_DELLOPRI | + apic_irq_to_idt(pin->io_vector); + } + mtx_lock_spin(&icu_lock); + ioapic_write(apic, IOAPIC_REDTBL_LO(i), flags); + + /* + * Route interrupts to the BSP by default using physical + * addressing. Vectored interrupts get readdressed using + * logical IDs to CPU clusters when they are enabled. + */ + flags = ioapic_read(apic, IOAPIC_REDTBL_HI(i)); + flags &= ~IOART_DEST; + KASSERT(PCPU_GET(apic_id) == lapic_id(), ("APIC ID mismatch")); /* XXXTEST */ + flags |= PCPU_GET(apic_id) << APIC_ID_SHIFT; + ioapic_write(apic, IOAPIC_REDTBL_HI(i), flags); + mtx_unlock_spin(&icu_lock); + if (pin->io_vector >= 0) { + #ifdef MIXED_MODE + /* Route IRQ0 via the 8259A using mixed mode. */ + if (pin->io_vector == 0) + ioapic_setup_mixed_mode(pin); + else + #endif + intr_register_source(&pin->io_intsrc); + } + + } + } + + /* + * Program all the intpins to use logical destinations once the AP's + * have been launched. + */ + static void + ioapic_set_logical_destinations(void *arg __unused) + { + struct ioapic *io; + int i; + + program_logical_dest = 1; + STAILQ_FOREACH(io, &ioapic_list, io_next) + for (i = 0; i < io->io_numintr; i++) + if (io->io_pins[i].io_dest != DEST_NONE && + io->io_pins[i].io_dest != DEST_EXTINT && + io->io_pins[i].io_vector != VECTOR_EXTINT /* XXXTEST */) + ioapic_program_destination(&io->io_pins[i]); + } + SYSINIT(ioapic_destinations, SI_SUB_SMP, SI_ORDER_SECOND, + ioapic_set_logical_destinations, NULL) + + #ifdef MIXED_MODE + /* + * Support for mixed-mode interrupt sources. These sources route an ISA + * IRQ through the 8259A's via the ExtINT on pin 0 of the I/O APIC that + * routes the ISA interrupts. We just ignore the intpins that use this + * mode and allow the atpic driver to register its interrupt source for + * that IRQ instead. + */ + + void + ioapic_setup_mixed_mode(struct ioapic_intsrc *intpin) + { + struct ioapic_intsrc *extint; + struct ioapic *io; + + /* + * Mark the associated I/O APIC intpin as being delivered via + * ExtINT and enable the ExtINT pin on the I/O APIC if needed. + */ + intpin->io_dest = DEST_EXTINT; + io = (struct ioapic *)intpin->io_intsrc.is_pic; + extint = &io->io_pins[0]; + if (extint->io_vector != VECTOR_EXTINT) + panic("Can't find ExtINT pin to route through!"); + #if 0 + /* XXXTEST? */ + if (extint->io_dest == DEST_NONE) + ioapic_assign_cluster(extint); + #endif + } + + #endif /* MIXED_MODE */ Index: i386/i386/local_apic.c =========================================================================== *** /dev/null Thu Oct 30 14:47:44 2003 --- i386/i386/local_apic.c Thu Oct 30 14:48:09 2003 *************** *** 0 **** --- 1,748 ---- + /*- + * Copyright (c) 2003 John Baldwin + * Copyright (c) 1996, by Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD: src/sys/i386/i386/mpapic.c,v 1.60 2002/09/21 19:03:42 markm Exp $ + */ + + /* + * Local APIC support on Pentium and later processors. + */ + + #include + #include + #include + #include + #include + + #include + #include + + #include + #include + #include + #include + #include + #include + #include + #include + + /* + * We can handle up to 60 APICs via our logical cluster IDs, but currently + * the physical IDs on Intel processors up to the Pentium 4 are limited to + * 16. + */ + #define MAX_APICID 16 + + /* + * Support for local APICs. Local APICs manage interrupts on each + * individual processor as opposed to I/O APICs which receive interrupts + * from I/O devices and then forward them on to the local APICs. + * + * Local APICs can also send interrupts to each other thus providing the + * mechanism for IPIs. + */ + + struct lvt { + u_int lvt_edgetrigger:1; + u_int lvt_activehi:1; + u_int lvt_masked:1; + u_int lvt_active:1; + u_int lvt_mode:16; + u_int lvt_vector:8; + }; + + struct lapic { + struct lvt la_lvts[LVT_MAX + 1]; + u_int la_id:8; + u_int la_cluster:4; + u_int la_cluster_id:2; + u_int la_present:1; + } static lapics[MAX_APICID]; + + /* XXX: should thermal be an NMI? */ + + /* Global defaults for local APIC LVT entries. */ + static struct lvt lvts[LVT_MAX + 1] = { + { 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 }, /* LINT0: masked ExtINT */ + { 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 }, /* LINT1: NMI */ + { 1, 1, 1, 1, APIC_LVT_DM_FIXED, 0 }, /* Timer: needs a vector */ + { 1, 1, 1, 1, APIC_LVT_DM_FIXED, 0 }, /* Error: needs a vector */ + { 1, 1, 1, 1, APIC_LVT_DM_FIXED, 0 }, /* PMC */ + { 1, 1, 1, 1, APIC_LVT_DM_FIXED, 0 }, /* Thermal: needs a vector */ + }; + + static inthand_t *ioint_handlers[] = { + NULL, /* 0 - 31 */ + IDTVEC(apic_isr1), /* 32 - 63 */ + IDTVEC(apic_isr2), /* 64 - 95 */ + IDTVEC(apic_isr3), /* 96 - 127 */ + IDTVEC(apic_isr4), /* 128 - 159 */ + IDTVEC(apic_isr5), /* 160 - 191 */ + NULL, /* 192 - 223 */ + NULL /* 224 - 255 */ + }; + + volatile lapic_t *lapic; + + static uint32_t + lvt_mode(struct lapic *la, u_int pin, uint32_t value) + { + struct lvt *lvt; + + KASSERT(pin <= LVT_MAX, ("%s: pin %u out of range", __func__, pin)); + if (la->la_lvts[pin].lvt_active) + lvt = &la->la_lvts[pin]; + else + lvt = &lvts[pin]; + + value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM | + APIC_LVT_VECTOR); + if (lvt->lvt_edgetrigger == 0) + value |= APIC_LVT_TM; + if (lvt->lvt_activehi == 0) + value |= APIC_LVT_IIPP_INTALO; + if (lvt->lvt_masked) + value |= APIC_LVT_M; + value |= lvt->lvt_mode; + switch (lvt->lvt_mode) { + case APIC_LVT_DM_NMI: + case APIC_LVT_DM_SMI: + case APIC_LVT_DM_INIT: + case APIC_LVT_DM_EXTINT: + if (!lvt->lvt_edgetrigger) { + printf("lapic%u: Forcing LINT%u to edge trigger\n", + la->la_id, pin); + value |= APIC_LVT_TM; + } + /* Use a vector of 0. */ + break; + case APIC_LVT_DM_FIXED: + #if 0 + value |= lvt->lvt_vector; + #else + panic("Fixed LINT pins not supported"); + #endif + break; + default: + panic("bad APIC LVT delivery mode: %#x\n", value); + } + return (value); + } + + /* + * Map the local APIC and setup necessary interrupt vectors. + */ + void + lapic_init(uintptr_t addr) + { + u_int32_t value; + + /* Map the local APIC and setup the spurious interrupt handler. */ + KASSERT(trunc_page(addr) == addr, + ("local APIC not aligned on a page boundary")); + lapic = (lapic_t *)pmap_mapdev(addr, sizeof(lapic_t)); + setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_SYS386IGT, SEL_KPL, + GSEL(GCODE_SEL, SEL_KPL)); + + /* Perform basic initialization of the BSP's local APIC. */ + value = lapic->svr; + value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS); + value |= (APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT); + lapic->svr = value; + + /* Set BSP's per-CPU local APIC ID. */ + PCPU_SET(apic_id, lapic_id()); + + /* XXX: timer/error/thermal interrupts */ + } + + /* + * Create a local APIC instance. + */ + void + lapic_create(u_int apic_id, int boot_cpu) + { + int i; + + if (apic_id > MAX_APICID) { + printf("APIC: Ignoring local APIC with ID %d\n", apic_id); + if (boot_cpu) + panic("Can't ignore BSP"); + return; + } + KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u", + apic_id)); + + /* + * Assume no local LVT overrides and a cluster of 0 and + * intra-cluster ID of 0. + */ + lapics[apic_id].la_present = 1; + lapics[apic_id].la_id = apic_id; + for (i = 0; i < LVT_MAX; i++) { + lapics[apic_id].la_lvts[i] = lvts[i]; + lapics[apic_id].la_lvts[i].lvt_active = 0; + } + + #ifdef SMP + cpu_add(apic_id, boot_cpu); + #endif + } + + /* + * Dump contents of local APIC registers + */ + void + lapic_dump(const char* str) + { + + printf("cpu%d %s:\n", PCPU_GET(cpuid), str); + printf(" ID: 0x%08x VER: 0x%08x LDR: 0x%08x DFR: 0x%08x\n", + lapic->id, lapic->version, lapic->ldr, lapic->dfr); + printf(" lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n", + lapic->lvt_lint0, lapic->lvt_lint1, lapic->tpr, lapic->svr); + } + + void + lapic_enable_intr(u_int irq) + { + u_int vector; + + vector = apic_irq_to_idt(irq); + KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); + KASSERT(ioint_handlers[vector / 32] != NULL, + ("No ISR handler for IRQ %u", irq)); + setidt(vector, ioint_handlers[vector / 32], SDT_SYS386IGT, SEL_KPL, + GSEL(GCODE_SEL, SEL_KPL)); + } + + void + lapic_setup(void) + { + struct lapic *la; + u_int32_t value, maxlvt; + register_t eflags; + + la = &lapics[lapic_id()]; + KASSERT(la->la_present, ("missing APIC structure")); + eflags = intr_disable(); + maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; + + /* Program LINT[01] LVT entries. */ + lapic->lvt_lint0 = lvt_mode(la, LVT_LINT0, lapic->lvt_lint0); + lapic->lvt_lint1 = lvt_mode(la, LVT_LINT1, lapic->lvt_lint1); + + /* XXX: more LVT entries */ + + /* Clear the TPR. */ + value = lapic->tpr; + value &= ~APIC_TPR_PRIO; + lapic->tpr = value; + + /* Use the cluster model for logical IDs. */ + value = lapic->dfr; + value &= ~APIC_DFR_MODEL_MASK; + value |= APIC_DFR_MODEL_CLUSTER; + lapic->dfr = value; + + /* Set this APIC's logical ID. */ + /* XXXTEST */ + printf("cpu %d: Programming physical ID %u to cluster %u, ID %u\n", + PCPU_GET(cpuid), lapic_id(), la->la_cluster, la->la_cluster_id); + value = lapic->ldr; + value &= ~APIC_ID_MASK; + value |= (la->la_cluster << APIC_ID_CLUSTER_SHIFT | + 1 << la->la_cluster_id) << APIC_ID_SHIFT; + lapic->ldr = value; + + /* Setup spurious vector and enable the local APIC. */ + value = lapic->svr; + value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS); + value |= (APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT); + lapic->svr = value; + intr_restore(eflags); + } + + void + lapic_disable(void) + { + uint32_t value; + + /* Software disable the local APIC. */ + value = lapic->svr; + value &= ~APIC_SVR_SWEN; + lapic->svr = value; + } + + int + lapic_id(void) + { + + KASSERT(lapic != NULL, ("local APIC is not mapped")); + return (lapic->id >> APIC_ID_SHIFT); + } + + int + lapic_intr_pending(u_int vector) + { + volatile u_int32_t *irr; + + /* + * The IRR registers are an array of 128-bit registers each of + * which only describes 32 interrupts in the low 32 bits.. Thus, + * we divide the vector by 32 to get the 128-bit index. We then + * multiply that index by 4 to get the equivalent index from + * treating the IRR as an array of 32-bit registers. Finally, we + * modulus the vector by 32 to determine the individual bit to + * test. + */ + irr = &lapic->irr0; + return (irr[(vector / 32) * 4] & 1 << (vector % 32)); + } + + void + lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) + { + struct lapic *la; + + KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist", + __func__, apic_id)); + KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big", + __func__, cluster)); + KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID, + ("%s: intra cluster id %u too big", __func__, cluster_id)); + la = &lapics[apic_id]; + la->la_cluster = cluster; + la->la_cluster_id = cluster_id; + } + + int + lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked) + { + + if (pin > LVT_MAX) + return (EINVAL); + if (apic_id == APIC_ID_ALL) + lvts[pin].lvt_masked = masked; + else { + KASSERT(lapics[apic_id].la_present, + ("%s: missing APIC %u", __func__, apic_id)); + lapics[apic_id].la_lvts[pin].lvt_masked = masked; + lapics[apic_id].la_lvts[pin].lvt_active = 1; + } + /* XXXTEST */ + printf("lapic%u: LINT%u %s\n", apic_id, pin, + masked ? "masked" : "unmasked"); + return (0); + } + + int + lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode) + { + struct lvt *lvt; + + if (pin > LVT_MAX) + return (EINVAL); + if (apic_id == APIC_ID_ALL) + lvt = &lvts[pin]; + else { + KASSERT(lapics[apic_id].la_present, + ("%s: missing APIC %u", __func__, apic_id)); + lvt = &lapics[apic_id].la_lvts[pin]; + lvt->lvt_active = 1; + + } + lvt->lvt_mode = mode; + switch (mode) { + case APIC_LVT_DM_NMI: + case APIC_LVT_DM_SMI: + case APIC_LVT_DM_INIT: + case APIC_LVT_DM_EXTINT: + lvt->lvt_edgetrigger = 1; + lvt->lvt_activehi = 1; + if (mode == APIC_LVT_DM_EXTINT) + lvt->lvt_masked = 1; + else + lvt->lvt_masked = 0; + break; + default: + panic("Unsupported delivery mode: 0x%x\n", mode); + } + /* XXXTEST */ + printf("lapic%u: Routing ", apic_id); + switch (mode) { + case APIC_LVT_DM_NMI: + printf("NMI"); + break; + case APIC_LVT_DM_SMI: + printf("SMI"); + break; + case APIC_LVT_DM_INIT: + printf("INIT"); + break; + case APIC_LVT_DM_EXTINT: + printf("ExtINT"); + break; + } + printf(" -> LINT%u\n", pin); + return (0); + } + + int + lapic_set_lvt_polarity(u_int apic_id, u_int pin, u_char activehi) + { + + if (pin > LVT_MAX) + return (EINVAL); + if (apic_id == APIC_ID_ALL) + lvts[pin].lvt_activehi = activehi; + else { + KASSERT(lapics[apic_id].la_present, + ("%s: missing APIC %u", __func__, apic_id)); + lapics[apic_id].la_lvts[pin].lvt_active = 1; + lapics[apic_id].la_lvts[pin].lvt_activehi = activehi; + } + /* XXXTEST */ + printf("lapic%u: LINT%u polarity: active-%s\n", apic_id, pin, + activehi ? "hi" : "lo"); + return (0); + } + + int + lapic_set_lvt_triggermode(u_int apic_id, u_int pin, u_char edgetrigger) + { + + if (pin > LVT_MAX) + return (EINVAL); + if (apic_id == APIC_ID_ALL) + lvts[pin].lvt_edgetrigger = edgetrigger; + else { + KASSERT(lapics[apic_id].la_present, + ("%s: missing APIC %u", __func__, apic_id)); + lapics[apic_id].la_lvts[pin].lvt_edgetrigger = edgetrigger; + lapics[apic_id].la_lvts[pin].lvt_active = 1; + } + /* XXXTEST */ + printf("lapic%u: LINT%u trigger: %s\n", apic_id, pin, + edgetrigger ? "edge" : "level"); + return (0); + } + + void + lapic_handle_intr(struct intrframe frame) + { + struct intsrc *isrc; + + if (frame.if_vec == -1) + panic("Couldn't get vector from ISR!"); + isrc = intr_lookup_source(apic_idt_to_irq(frame.if_vec)); + isrc->is_pic->pic_disable_source(isrc); + lapic->eoi = 0; + intr_execute_handlers(isrc, &frame); + } + + /* Translate between IDT vectors and IRQ vectors. */ + u_int + apic_irq_to_idt(u_int irq) + { + u_int vector; + + KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq)); + vector = irq + IDT_IO_INTS; + if (vector >= IDT_SYSCALL) + vector++; + return (vector); + } + + u_int + apic_idt_to_irq(u_int vector) + { + + KASSERT(vector >= IDT_IO_INTS && vector != IDT_SYSCALL && + vector <= IDT_IO_INTS + NUM_IO_INTS, + ("Vector %u does not map to an IRQ line", vector)); + if (vector > IDT_SYSCALL) + vector--; + return (vector - IDT_IO_INTS); + } + + /* + * APIC probing support code. This includes code to manage enumerators. + */ + + static SLIST_HEAD(, apic_enumerator) enumerators = + SLIST_HEAD_INITIALIZER(enumerators); + static struct apic_enumerator *best_enum; + + void + apic_register_enumerator(struct apic_enumerator *enumerator) + { + #ifdef INVARIANTS + struct apic_enumerator *apic_enum; + + SLIST_FOREACH(apic_enum, &enumerators, apic_next) { + if (apic_enum == enumerator) + panic("%s: Duplicate register of %s", __func__, + enumerator->apic_name); + } + #endif + SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next); + } + + /* + * We have to look for CPU's very, very early because certain subsystems + * want to know how many CPU's we have extremely early on in the boot + * process. + */ + static void + apic_init(void *dummy __unused) + { + struct apic_enumerator *enumerator; + int retval, best; + + /* We only support built in local APICs. */ + if (!(cpu_feature & CPUID_APIC)) + return; + + /* First, probe all the enumerators to find the best match. */ + best_enum = NULL; + best = 0; + SLIST_FOREACH(enumerator, &enumerators, apic_next) { + retval = enumerator->apic_probe(); + if (retval > 0) + continue; + if (best_enum == NULL || best < retval) { + best_enum = enumerator; + best = retval; + } + } + if (best_enum == NULL) { + #if 0 + if (bootverbose) + #endif + printf("APIC: Could not find any APICs.\n"); + return; + } + + #if 0 + if (bootverbose) + #endif + printf("APIC: Using the %s enumerator.\n", + best_enum->apic_name); + + /* Second, probe the CPU's in the system. */ + retval = best_enum->apic_probe_cpus(); + if (retval != 0) + printf("%s: Failed to probe CPUs: returned %d\n", + best_enum->apic_name, retval); + } + SYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL) + + /* + * Setup the local APIC. We have to do this prior to starting up the APs + * in the SMP case. + */ + static void + apic_setup_local(void *dummy __unused) + { + int retval; + uint64_t apic_base; + + if (best_enum == NULL) + return; + /* + * To work around an errata, we disable the local APIC on some + * CPUs during early startup. We need to turn the local APIC back + * on on such CPUs now. + */ + if (cpu == CPU_686 && strcmp(cpu_vendor, "GenuineIntel") == 0 && + (cpu_id & 0xff0) == 0x610) { + apic_base = rdmsr(MSR_APICBASE); + apic_base |= APICBASE_ENABLED; + wrmsr(MSR_APICBASE, apic_base); + } + retval = best_enum->apic_setup_local(); + if (retval != 0) + printf("%s: Failed to setup the local APIC: returned %d\n", + best_enum->apic_name, retval); + } + SYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_FIRST, apic_setup_local, NULL) + + /* + * Setup the I/O APICs. + */ + static void + apic_setup_io(void *dummy __unused) + { + int retval; + + if (best_enum == NULL) + return; + retval = best_enum->apic_setup_io(); + if (retval != 0) + printf("%s: Failed to setup I/O APICs: returned %d\n", + best_enum->apic_name, retval); + + /* + * Finish setting up the local APIC on the BSP once we know how to + * properly program the LINT pins. + */ + lapic_setup(); + #if 0 + if (bootverbose) + #endif + lapic_dump("BSP"); + } + SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_SECOND, apic_setup_io, NULL) + + #ifdef SMP + /* + * Inter Processor Interrupt functions. The lapic_ipi_*() functions are + * private the sys/i386 code. The public interface for the rest of the + * kernel is defined in mp_machdep.c. + */ + #define DETECT_DEADLOCK + + int + lapic_ipi_wait(int delay) + { + int x, incr; + + /* + * Wait delay loops for IPI to be sent. This is highly bogus + * since this is sensitive to CPU clock speed. If delay is + * -1, we wait forever. + */ + if (delay == -1) { + incr = 0; + delay = 1; + } else + incr = 1; + for (x = 0; x < delay; x += incr) { + if ((lapic->icr_lo & APIC_DELSTAT_MASK) == APIC_DELSTAT_IDLE) + return (1); + ia32_pause(); + } + return (0); + } + + void + lapic_ipi_raw(register_t icrlo, u_int dest) + { + register_t value, eflags; + + /* XXX: Need more sanity checking of icrlo? */ + KASSERT(lapic != NULL, ("%s called too early", __func__)); + KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, + ("%s: invalid dest field", __func__)); + KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0, + ("%s: reserved bits set in ICR LO register", __func__)); + + /* Set destination in ICR HI register if it is being used. */ + eflags = intr_disable(); + if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) { + value = lapic->icr_hi; + value &= ~APIC_ID_MASK; + value |= dest << APIC_ID_SHIFT; + lapic->icr_hi = value; + } + + /* Program the contents of the IPI and dispatch it. */ + value = lapic->icr_lo; + value &= APIC_ICRLO_RESV_MASK; + value |= icrlo; + lapic->icr_lo = value; + intr_restore(eflags); + } + + #ifdef DETECT_DEADLOCK + #define BEFORE_SPIN 1000000 + #define AFTER_SPIN 1000 + #endif + + void + lapic_ipi_vectored(u_int vector, int dest) + { + register_t icrlo, destfield; + + KASSERT((vector & ~APIC_VECTOR_MASK) == 0, + ("%s: invalid vector %d", __func__, vector)); + + icrlo = vector | APIC_DELMODE_FIXED | APIC_DESTMODE_PHY | + APIC_LEVEL_DEASSERT | APIC_TRIGMOD_EDGE; + destfield = 0; + switch (dest) { + case APIC_IPI_DEST_SELF: + icrlo |= APIC_DEST_SELF; + break; + case APIC_IPI_DEST_ALL: + icrlo |= APIC_DEST_ALLISELF; + break; + case APIC_IPI_DEST_OTHERS: + icrlo |= APIC_DEST_ALLESELF; + break; + default: + KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, + ("%s: invalid destination 0x%x", __func__, dest)); + destfield = dest; + } + + #ifdef DETECT_DEADLOCK + /* Check for an earlier stuck IPI. */ + if (!lapic_ipi_wait(BEFORE_SPIN)) + panic("APIC: Previous IPI is stuck"); + #endif + + lapic_ipi_raw(icrlo, destfield); + + #ifdef DETECT_DEADLOCK + /* Wait for IPI to be delivered. */ + if (!lapic_ipi_wait(AFTER_SPIN)) { + #ifdef needsattention + /* + * XXX FIXME: + * + * The above function waits for the message to actually be + * delivered. It breaks out after an arbitrary timeout + * since the message should eventually be delivered (at + * least in theory) and that if it wasn't we would catch + * the failure with the check above when the next IPI is + * sent. + * + * We could skiip this wait entirely, EXCEPT it probably + * protects us from other routines that assume that the + * message was delivered and acted upon when this function + * returns. + */ + printf("APIC: IPI might be stuck\n"); + #else /* !needsattention */ + /* Wait until mesage is sent without a timeout. */ + while (lapic->icr_lo & APIC_DELSTAT_PEND) + ia32_pause(); + #endif /* needsattention */ + } + #endif /* DETECT_DEADLOCK */ + } + #endif /* SMP */ Index: i386/i386/locore.s =========================================================================== --- sys/i386/i386/locore.s 2003/10/30 14:48:07 #36 +++ i386/i386/locore.s 2003/10/30 14:48:07 @@ -82,9 +82,8 @@ * This is "constructed" in locore.s on the BSP and in mp_machdep.c * for each AP. DO NOT REORDER THESE WITHOUT UPDATING THE REST! */ - .globl SMP_prvspace, lapic + .globl SMP_prvspace .set SMP_prvspace,(MPPTDI << PDRSHIFT) - .set lapic,SMP_prvspace + (NPTEPG-1) * PAGE_SIZE #endif /* SMP */ /* Index: i386/i386/machdep.c =========================================================================== --- sys/i386/i386/machdep.c 2003/10/30 14:48:07 #116 +++ i386/i386/machdep.c 2003/10/30 14:48:07 @@ -40,6 +40,7 @@ #include __FBSDID("$FreeBSD: src/sys/i386/i386/machdep.c,v 1.578 2003/10/30 21:42:16 jhb Exp $"); +#include "opt_apic.h" #include "opt_atalk.h" #include "opt_compat.h" #include "opt_cpu.h" @@ -92,7 +93,10 @@ #include #include +#ifdef DDB #include +#include +#endif #include @@ -102,6 +106,7 @@ #include #include #include +#include #include #include #include /* pcb.h included via sys/user.h */ @@ -114,8 +119,10 @@ #include #endif +#ifdef DEV_ISA #include -#include +#endif + #include #include #include @@ -150,7 +157,7 @@ SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL) int _udatasel, _ucodesel; -u_int atdevbase; +u_int atdevbase, basemem; int cold = 1; @@ -224,10 +231,7 @@ bufinit(); vm_pager_bufferinit(); -#ifndef SMP - /* For SMP, we delay the cpu_setregs() until after SMP startup. */ cpu_setregs(); -#endif } /* @@ -1467,6 +1471,31 @@ IDTVEC(page), IDTVEC(mchk), IDTVEC(rsvd), IDTVEC(fpu), IDTVEC(align), IDTVEC(xmm), IDTVEC(lcall_syscall), IDTVEC(int0x80_syscall); +#ifdef DDB +/* + * Display the index and function name of any IDT entries that don't use + * the default 'rsvd' entry point. + */ +DB_SHOW_COMMAND(idt, db_show_idt) +{ + struct gate_descriptor *ip; + int idx, quit; + uintptr_t func; + + ip = idt; + db_setup_paging(db_simple_pager, &quit, DB_LINES_PER_PAGE); + for (idx = 0, quit = 0; idx < NIDT; idx++) { + func = (ip->gd_hioffset << 16 | ip->gd_looffset); + if (func != (uintptr_t)&IDTVEC(rsvd)) { + db_printf("%3d\t", idx); + db_printsym(func, DB_STGY_PROC); + db_printf("\n"); + } + ip++; + } +} +#endif + void sdtossd(sd, ssd) struct segment_descriptor *sd; @@ -1501,7 +1530,7 @@ { int i, physmap_idx, pa_indx; int hasbrokenint12; - u_int basemem, extmem; + u_int extmem; struct vm86frame vmf; struct vm86context vmc; vm_paddr_t pa, physmap[PHYSMAP_SIZE]; @@ -1719,10 +1748,7 @@ #ifdef SMP /* make hole for AP bootstrap code */ - physmap[1] = mp_bootaddress(physmap[1] / 1024); - - /* look for the MP hardware - needed for apic addresses */ - i386_mp_probe(); + physmap[1] = mp_bootaddress(physmap[1]); #endif /* @@ -2065,7 +2091,7 @@ printf("WARNING: loader(8) metadata is missing!\n"); #ifdef DEV_ISA - isa_defaultirq(); + atpic_startup(); #endif #ifdef DDB @@ -2724,6 +2750,90 @@ } #endif /* no DDB */ +#ifndef DEV_APIC +#include + +/* + * Provide stub functions so that the MADT APIC enumerator in the acpi + * kernel module will link against a kernel without 'device apic'. + * + * XXX - This is a gross hack. + */ +void +apic_register_enumerator(struct apic_enumerator *enumerator) +{ +} + +void * +ioapic_create(uintptr_t addr, int32_t id, int intbase) +{ + return (NULL); +} + +void +ioapic_register(void *cookie) +{ +} + +int +ioapic_remap_vector(void *cookie, u_int pin, int vector) +{ + return (ENXIO); +} + +int +ioapic_set_extint(void *cookie, u_int pin) +{ + return (ENXIO); +} + +int +ioapic_set_nmi(void *cookie, u_int pin) +{ + return (ENXIO); +} + +int +ioapic_set_polarity(void *cookie, u_int pin, char activehi) +{ + return (ENXIO); +} + +int +ioapic_set_triggermode(void *cookie, u_int pin, char edgetrigger) +{ + return (ENXIO); +} + +void +lapic_create(u_int apic_id, int boot_cpu) +{ +} + +void +lapic_init(uintptr_t addr) +{ +} + +int +lapic_set_lvt_mode(u_int apic_id, u_int lvt, u_int32_t mode) +{ + return (ENXIO); +} + +int +lapic_set_lvt_polarity(u_int apic_id, u_int lvt, u_char activehi) +{ + return (ENXIO); +} + +int +lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, u_char edgetrigger) +{ + return (ENXIO); +} +#endif + #ifdef DDB /* Index: i386/i386/mp_machdep.c =========================================================================== --- sys/i386/i386/mp_machdep.c 2003/10/30 14:48:07 #60 +++ i386/i386/mp_machdep.c 2003/10/30 14:48:07 @@ -26,16 +26,25 @@ #include __FBSDID("$FreeBSD: src/sys/i386/i386/mp_machdep.c,v 1.219 2003/10/30 21:42:16 jhb Exp $"); +#include "opt_apic.h" #include "opt_cpu.h" #include "opt_kstack_pages.h" -#ifdef SMP -#include -#else #if !defined(lint) -#error +#if !defined(SMP) +#error How did you get here? +#endif + +#if defined(I386_CPU) && !defined(COMPILING_LINT) +#error SMP not supported with I386_CPU #endif +#ifndef DEV_APIC +#error The apic device is required for SMP, add "device apic" to your config file. #endif +#if defined(CPU_DISABLE_CMPXCHG) && !defined(COMPILING_LINT) +#error SMP not supported with CPU_DISABLE_CMPXCHG +#endif +#endif /* not lint */ #include #include @@ -54,138 +63,31 @@ #include #include #include -#include #include #include #include #include #include -#include -#include -#include +#include #include -#include -#include -#include -#include -#include +#include +#include #include -#include /** TEST_DEFAULT_CONFIG, TEST_TEST1 */ -#include +#include /** COUNT_XINVLTLB_HITS, USE_COMLOCK */ #include #include -#if defined(APIC_IO) -#include /* setidt() */ -#include /* IPIs */ -#include /* IPIs */ -#endif /* APIC_IO */ - -#if defined(TEST_DEFAULT_CONFIG) -#define MPFPS_MPFB1 TEST_DEFAULT_CONFIG -#else -#define MPFPS_MPFB1 mpfps->mpfb1 -#endif /* TEST_DEFAULT_CONFIG */ - #define WARMBOOT_TARGET 0 #define WARMBOOT_OFF (KERNBASE + 0x0467) #define WARMBOOT_SEG (KERNBASE + 0x0469) -#ifdef PC98 -#define BIOS_BASE (0xe8000) -#define BIOS_SIZE (0x18000) -#else -#define BIOS_BASE (0xf0000) -#define BIOS_SIZE (0x10000) -#endif -#define BIOS_COUNT (BIOS_SIZE/4) - #define CMOS_REG (0x70) #define CMOS_DATA (0x71) #define BIOS_RESET (0x0f) #define BIOS_WARM (0x0a) -#define PROCENTRY_FLAG_EN 0x01 -#define PROCENTRY_FLAG_BP 0x02 -#define IOAPICENTRY_FLAG_EN 0x01 - - -/* MP Floating Pointer Structure */ -typedef struct MPFPS { - char signature[4]; - void *pap; - u_char length; - u_char spec_rev; - u_char checksum; - u_char mpfb1; - u_char mpfb2; - u_char mpfb3; - u_char mpfb4; - u_char mpfb5; -} *mpfps_t; - -/* MP Configuration Table Header */ -typedef struct MPCTH { - char signature[4]; - u_short base_table_length; - u_char spec_rev; - u_char checksum; - u_char oem_id[8]; - u_char product_id[12]; - void *oem_table_pointer; - u_short oem_table_size; - u_short entry_count; - void *apic_address; - u_short extended_table_length; - u_char extended_table_checksum; - u_char reserved; -} *mpcth_t; - - -typedef struct PROCENTRY { - u_char type; - u_char apic_id; - u_char apic_version; - u_char cpu_flags; - u_long cpu_signature; - u_long feature_flags; - u_long reserved1; - u_long reserved2; -} *proc_entry_ptr; - -typedef struct BUSENTRY { - u_char type; - u_char bus_id; - char bus_type[6]; -} *bus_entry_ptr; - -typedef struct IOAPICENTRY { - u_char type; - u_char apic_id; - u_char apic_version; - u_char apic_flags; - void *apic_address; -} *io_apic_entry_ptr; - -typedef struct INTENTRY { - u_char type; - u_char int_type; - u_short int_flags; - u_char src_bus_id; - u_char src_bus_irq; - u_char dst_apic_id; - u_char dst_apic_int; -} *int_entry_ptr; - -/* descriptions of MP basetable entries */ -typedef struct BASETABLE_ENTRY { - u_char type; - u_char length; - char name[16]; -} basetable_entry; - /* * this code MUST be enabled here and in mpboot.s. * it follows the very early stages of AP boot by placing values in CMOS ram. @@ -220,6 +122,7 @@ #define CHECK_INIT(D) #define CHECK_PRINT(S) +#define CHECK_WRITE(A, D) #endif /* CHECK_POINTS */ @@ -240,53 +143,28 @@ #define MP_ANNOUNCE_POST 0x19 -static int need_hyperthreading_fixup; -static u_int logical_cpus; -static u_int logical_cpus_mask; +/* lock region used by kernel profiling */ +int mcount_lock; -/* used to hold the AP's until we are ready to release them */ -static struct mtx ap_boot_mtx; +#ifdef USE_COMLOCK +/* locks com (tty) data/hardware accesses: a FASTINTR() */ +struct mtx com_mtx; +#endif /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ int current_postcode; -int bsp_apic_ready = 0; /* flags useability of BSP apic */ int mp_naps; /* # of Applications processors */ -int mp_nbusses; /* # of busses */ -int mp_napics; /* # of IO APICs */ -int boot_cpu_id; /* designated BSP */ -vm_offset_t cpu_apic_address; -vm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */ +int boot_cpu_id = -1; /* designated BSP */ extern int nkpt; -u_int32_t cpu_apic_versions[MAXCPU]; -u_int32_t *io_apic_versions; - -#ifdef APIC_INTR_REORDER -struct { - volatile int *location; - int bit; -} apic_isrbit_location[32]; -#endif - -struct apic_intmapinfo int_to_apicintpin[APIC_INTMAPSIZE]; - /* - * APIC ID logical/physical mapping structures. - * We oversize these to simplify boot-time config. + * CPU topology map datastructures for HTT. (XXX) */ -int cpu_num_to_apic_id[NAPICID]; -int io_num_to_apic_id[NAPICID]; -int apic_id_to_logical[NAPICID]; - -/* - * CPU topology map datastructures for HTT. - */ -struct cpu_group mp_groups[NAPICID]; -struct cpu_top mp_top; -struct cpu_top *smp_topology; +struct cpu_group mp_groups[MAXCPU]; +struct cpu_top mp_top; +struct cpu_top *smp_topology; - /* AP uses this during bootstrap. Do not staticize. */ char *bootSTK; static int bootAP; @@ -299,42 +177,41 @@ struct pcb stoppcbs[MAXCPU]; -#ifdef APIC_IO /* Variables needed for SMP tlb shootdown. */ vm_offset_t smp_tlb_addr1; vm_offset_t smp_tlb_addr2; volatile int smp_tlb_wait; -static struct mtx smp_tlb_mtx; -#endif +struct mtx smp_tlb_mtx; /* * Local data and functions. */ +static u_int logical_cpus; +static u_int logical_cpus_mask; + +/* used to hold the AP's until we are ready to release them */ +static struct mtx ap_boot_mtx; + /* Set to 1 once we're ready to let the APs out of the pen. */ static volatile int aps_ready = 0; -static int mp_capable; -static u_int boot_address; -static u_int base_memory; +/* + * Store data from cpu_add() until later in the boot when we actually setup + * the APs. + */ +struct cpu_info { + int cpu_present:1; + int cpu_bsp:1; +} static cpu_info[MAXCPU]; +static int cpu_apic_ids[MAXCPU]; -static int picmode; /* 0: virtual wire mode, 1: PIC mode */ -static mpfps_t mpfps; -static int search_for_sig(u_int32_t target, int count); -static void mp_enable(u_int boot_addr); +static u_int boot_address; -static void mptable_hyperthread_fixup(u_int id_mask); -static void mptable_pass1(void); -static int mptable_pass2(void); -static void default_mp_table(int type); -static void fix_mp_table(void); -static void setup_apic_irq_mapping(void); -static void init_locks(void); +static void set_logical_apic_ids(void); static int start_all_aps(u_int boot_addr); static void install_ap_tramp(u_int boot_addr); -static int start_ap(int logicalCpu, u_int boot_addr); -void ap_init(void); -static int apic_int_is_bus_type(int intr, int bus_type); +static int start_ap(int apic_id, u_int boot_addr); static void release_aps(void *dummy); static int hlt_cpus_mask; @@ -342,30 +219,6 @@ static struct sysctl_ctx_list logical_cpu_clist; /* - * initialize all the SMP locks - */ - -/* lock region used by kernel profiling */ -int mcount_lock; - -#ifdef USE_COMLOCK -/* locks com (tty) data/hardware accesses: a FASTINTR() */ -struct mtx com_mtx; -#endif /* USE_COMLOCK */ - -static void -init_locks(void) -{ - -#ifdef USE_COMLOCK - mtx_init(&com_mtx, "com", NULL, MTX_SPIN); -#endif /* USE_COMLOCK */ -#ifdef APIC_IO - mtx_init(&smp_tlb_mtx, "tlb", NULL, MTX_SPIN); -#endif -} - -/* * Calculate usable address in base memory for AP trampoline code. */ u_int @@ -373,87 +226,151 @@ { POSTCODE(MP_BOOTADDRESS_POST); - base_memory = basemem * 1024; /* convert to bytes */ - - boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ - if ((base_memory - boot_address) < bootMP_size) + boot_address = basemem & ~0xfff; /* round down to 4k boundary */ + if ((basemem - boot_address) < bootMP_size) boot_address -= 4096; /* not enough, lower by 4k */ return boot_address; } - -/* - * Look for an Intel MP spec table (ie, SMP capable hardware). - */ void -i386_mp_probe(void) +cpu_add(u_int apic_id, char boot_cpu) { - int x; - u_long segment; - u_int32_t target; - POSTCODE(MP_PROBE_POST); - - /* see if EBDA exists */ - if ((segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) != 0) { - /* search first 1K of EBDA */ - target = (u_int32_t) (segment << 4); - if ((x = search_for_sig(target, 1024 / 4)) >= 0) - goto found; - } else { - /* last 1K of base memory, effective 'top of base' passed in */ - target = (u_int32_t) (base_memory - 0x400); - if ((x = search_for_sig(target, 1024 / 4)) >= 0) - goto found; + if (apic_id > MAXCPU) { + printf("SMP: CPU %d exceeds maximum CPU %d, ignoring\n", + apic_id, MAXCPU); + return; + } + KASSERT(cpu_info[apic_id].cpu_present == 0, ("CPU %d added twice", + apic_id)); + cpu_info[apic_id].cpu_present = 1; + if (boot_cpu) { + KASSERT(boot_cpu_id == -1, + ("CPU %d claims to be BSP, but CPU %d already is", apic_id, + boot_cpu_id)); + boot_cpu_id = apic_id; + cpu_info[apic_id].cpu_bsp = 1; } - - /* search the BIOS */ - target = (u_int32_t) BIOS_BASE; - if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) - goto found; - - /* nothing found */ - mpfps = (mpfps_t)0; - mp_capable = 0; - return; - -found: - /* calculate needed resources */ - mpfps = (mpfps_t)x; - mptable_pass1(); - - /* flag fact that we are running multiple processors */ - mp_capable = 1; + mp_ncpus++; + if (apic_id > mp_maxid) + mp_maxid = apic_id; +#if 0 + if (bootverbose) +#endif + printf("SMP: Added CPU %d (%s)\n", apic_id, boot_cpu ? "BSP" : + "AP"); + } int cpu_mp_probe(void) { + /* - * Record BSP in CPU map - * This is done here so that MBUF init code works correctly. + * Always record BSP in CPU map so that the mbuf init code works + * correctly. */ all_cpus = 1; + if (mp_ncpus == 0) { + /* + * No CPUs were found, so this must be a UP system. Setup + * the variables to represent a system with a single CPU + * with an id of 0. + */ + KASSERT(mp_maxid == 0, + ("%s: mp_ncpus is zero, but mp_maxid is not", __func__)); + mp_ncpus = 1; + return (0); + } + + /* At least one CPU was found. */ + if (mp_ncpus == 1) { + /* + * One CPU was found, so this must be a UP system with + * an I/O APIC. + */ + mp_maxid = 0; + return (0); + } - return (mp_capable); + /* At least two CPUs were found. */ + KASSERT(mp_maxid >= mp_ncpus - 1, + ("%s: counters out of sync: max %d, count %d", __func__, mp_maxid, + mp_ncpus)); + return (1); } /* - * Initialize the SMP hardware and the APIC and start up the AP's. + * Initialize the IPI handlers and start up the AP's. */ void cpu_mp_start(void) { + int i; + POSTCODE(MP_START_POST); - /* look for MP capable motherboard */ - if (mp_capable) - mp_enable(boot_address); - else - panic("MP hardware not found!"); + /* Initialize the logical ID to APIC ID table. */ + for (i = 0; i < MAXCPU; i++) + cpu_apic_ids[i] = -1; + + /* Install an inter-CPU IPI for TLB invalidation */ + setidt(IPI_INVLTLB, IDTVEC(invltlb), + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + setidt(IPI_INVLPG, IDTVEC(invlpg), + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + setidt(IPI_INVLRNG, IDTVEC(invlrng), + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + + /* Install an inter-CPU IPI for forwarding hardclock() */ + setidt(IPI_HARDCLOCK, IDTVEC(hardclock), + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + + /* Install an inter-CPU IPI for forwarding statclock() */ + setidt(IPI_STATCLOCK, IDTVEC(statclock), + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + + /* Install an inter-CPU IPI for lazy pmap release */ + setidt(IPI_LAZYPMAP, IDTVEC(lazypmap), + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + + /* Install an inter-CPU IPI for all-CPU rendezvous */ + setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous), + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + + /* Install an inter-CPU IPI for forcing an additional software trap */ + setidt(IPI_AST, IDTVEC(cpuast), + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + + /* Install an inter-CPU IPI for CPU stop/restart */ + setidt(IPI_STOP, IDTVEC(cpustop), + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + +#ifdef USE_COMLOCK + mtx_init(&com_mtx, "com", NULL, MTX_SPIN); +#endif + mtx_init(&smp_tlb_mtx, "tlb", NULL, MTX_SPIN); + + /* Set boot_cpu_id if needed. */ + KASSERT(PCPU_GET(apic_id) == lapic_id(), ("APIC ID mismatch")); /* XXXTEST */ + if (boot_cpu_id == -1) { + boot_cpu_id = PCPU_GET(apic_id); + cpu_info[boot_cpu_id].cpu_bsp = 1; + } else + KASSERT(boot_cpu_id == PCPU_GET(apic_id), + ("BSP's APIC ID doesn't match boot_cpu_id")); + cpu_apic_ids[0] = boot_cpu_id; + + /* Start each Application Processor */ + start_all_aps(boot_address); + + /* Setup the initial logical CPUs info. */ + logical_cpus = logical_cpus_mask = 0; + if (cpu_feature & CPUID_HTT) + logical_cpus = (cpu_procinfo & CPUID_HTT_CORES) >> 16; - cpu_setregs(); + set_logical_apic_ids(); } @@ -463,40 +380,33 @@ void cpu_mp_announce(void) { - int x; + int i, x; POSTCODE(MP_ANNOUNCE_POST); - printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0)); - printf(", version: 0x%08x", cpu_apic_versions[0]); - printf(", at 0x%08x\n", cpu_apic_address); - for (x = 1; x <= mp_naps; ++x) { - printf(" cpu%d (AP): apic id: %2d", x, CPU_TO_ID(x)); - printf(", version: 0x%08x", cpu_apic_versions[x]); - printf(", at 0x%08x\n", cpu_apic_address); + /* List CPUs */ + printf(" cpu0 (BSP): APIC ID: %2d\n", boot_cpu_id); + for (i = 1, x = 0; x < MAXCPU; x++) { + if (cpu_info[x].cpu_present && !cpu_info[x].cpu_bsp) { + KASSERT(i < mp_ncpus, + ("mp_ncpus and actual cpus are out of whack")); + printf(" cpu%d (AP): APIC ID: %2d\n", i++, x); + } } - -#if defined(APIC_IO) - for (x = 0; x < mp_napics; ++x) { - printf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x)); - printf(", version: 0x%08x", io_apic_versions[x]); - printf(", at 0x%08x\n", io_apic_address[x]); - } -#else - printf(" Warning: APIC I/O disabled\n"); -#endif /* APIC_IO */ } /* - * AP cpu's call this to sync up protected mode. + * AP CPU's call this to initialize themselves. */ void init_secondary(void) { int gsel_tss; - int x, myid = bootAP; + int x, myid; u_int cr0; + /* bootAP is set in start_ap() to our ID. */ + myid = bootAP; gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid]; gdt_segs[GPROC0_SEL].ssd_base = (int) &SMP_prvspace[myid].pcpu.pc_common_tss; @@ -533,1547 +443,151 @@ cr0 = rcr0(); cr0 &= ~(CR0_CD | CR0_NW | CR0_EM); load_cr0(cr0); -} + CHECK_WRITE(0x38, 5); + + /* Disable local APIC just to be sure. */ + lapic_disable(); + /* signal our startup to the BSP. */ + mp_naps++; + CHECK_WRITE(0x39, 6); -#if defined(APIC_IO) -/* - * Final configuration of the BSP's local APIC: - * - disable 'pic mode'. - * - disable 'virtual wire mode'. - * - enable NMI. - */ -void -bsp_apic_configure(void) -{ - u_char byte; - u_int32_t temp; + /* Spin until the BSP releases the AP's. */ + while (!aps_ready) + ia32_pause(); - /* leave 'pic mode' if necessary */ - if (picmode) { - outb(0x22, 0x70); /* select IMCR */ - byte = inb(0x23); /* current contents */ - byte |= 0x01; /* mask external INTR */ - outb(0x23, byte); /* disconnect 8259s/NMI */ - } - - /* mask lint0 (the 8259 'virtual wire' connection) */ - temp = lapic.lvt_lint0; - temp |= APIC_LVT_M; /* set the mask */ - lapic.lvt_lint0 = temp; - - /* setup lint1 to handle NMI */ - temp = lapic.lvt_lint1; - temp &= ~APIC_LVT_M; /* clear the mask */ - lapic.lvt_lint1 = temp; - - if (bootverbose) - apic_dump("bsp_apic_configure()"); -} -#endif /* APIC_IO */ - - -/******************************************************************* - * local functions and data - */ - -/* - * start the SMP system - */ -static void -mp_enable(u_int boot_addr) -{ - int x; -#if defined(APIC_IO) - int apic; - u_int ux; -#endif /* APIC_IO */ - - POSTCODE(MP_ENABLE_POST); - - /* turn on 4MB of V == P addressing so we can get to MP table */ - *(int *)PTD = PG_V | PG_RW | ((uintptr_t)(void *)KPTphys & PG_FRAME); + /* BSP may have changed PTD while we were waiting */ invltlb(); + pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1); - /* examine the MP table for needed info, uses physical addresses */ - x = mptable_pass2(); +#if defined(I586_CPU) && !defined(NO_F00F_HACK) + lidt(&r_idt); +#endif - *(int *)PTD = 0; - invltlb(); + /* set up CPU registers and state */ + cpu_setregs(); - /* can't process default configs till the CPU APIC is pmapped */ - if (x) - default_mp_table(x); + /* set up FPU state on the AP */ + npxinit(__INITIAL_NPXCW__); - /* post scan cleanup */ - fix_mp_table(); - setup_apic_irq_mapping(); + /* set up SSE registers */ + enable_sse(); -#if defined(APIC_IO) - - /* fill the LOGICAL io_apic_versions table */ - for (apic = 0; apic < mp_napics; ++apic) { - ux = io_apic_read(apic, IOAPIC_VER); - io_apic_versions[apic] = ux; - io_apic_set_id(apic, IO_TO_ID(apic)); + /* A quick check from sanity claus */ + if (PCPU_GET(apic_id) != lapic_id()) { + printf("SMP: cpuid = %d\n", PCPU_GET(cpuid)); + printf("SMP: actual apic_id = %d\n", lapic_id()); + printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id)); + printf("PTD[MPPTDI] = %#jx\n", (uintmax_t)PTD[MPPTDI]); + panic("cpuid mismatch! boom!!"); } - /* program each IO APIC in the system */ - for (apic = 0; apic < mp_napics; ++apic) - if (io_apic_setup(apic) < 0) - panic("IO APIC setup failure"); + mtx_lock_spin(&ap_boot_mtx); - /* install a 'Spurious INTerrupt' vector */ - setidt(XSPURIOUSINT_OFFSET, Xspuriousint, - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + /* Init local apic for irq's */ + lapic_setup(); - /* install an inter-CPU IPI for TLB invalidation */ - setidt(XINVLTLB_OFFSET, Xinvltlb, - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - setidt(XINVLPG_OFFSET, Xinvlpg, - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - setidt(XINVLRNG_OFFSET, Xinvlrng, - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + /* Set memory range attributes for this CPU to match the BSP */ + mem_range_AP_init(); - /* install an inter-CPU IPI for forwarding hardclock() */ - setidt(XHARDCLOCK_OFFSET, Xhardclock, - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - - /* install an inter-CPU IPI for forwarding statclock() */ - setidt(XSTATCLOCK_OFFSET, Xstatclock, - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - - /* install an inter-CPU IPI for lazy pmap release */ - setidt(XLAZYPMAP_OFFSET, Xlazypmap, - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + smp_cpus++; - /* install an inter-CPU IPI for all-CPU rendezvous */ - setidt(XRENDEZVOUS_OFFSET, Xrendezvous, - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); + printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); - /* install an inter-CPU IPI for forcing an additional software trap */ - setidt(XCPUAST_OFFSET, Xcpuast, - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + /* Determine if we are a logical CPU. */ + if (logical_cpus > 1 && PCPU_GET(apic_id) % logical_cpus != 0) + logical_cpus_mask |= PCPU_GET(cpumask); + + /* Build our map of 'other' CPUs. */ + PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); - /* install an inter-CPU IPI for CPU stop/restart */ - setidt(XCPUSTOP_OFFSET, Xcpustop, - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#if 0 + if (bootverbose) +#endif + lapic_dump("AP"); -#if defined(TEST_TEST1) - /* install a "fake hardware INTerrupt" vector */ - setidt(XTEST1_OFFSET, Xtest1, - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); -#endif /** TEST_TEST1 */ - -#endif /* APIC_IO */ - - /* initialize all SMP locks */ - init_locks(); - - /* start each Application Processor */ - start_all_aps(boot_addr); -} - - -/* - * look for the MP spec signature - */ - -/* string defined by the Intel MP Spec as identifying the MP table */ -#define MP_SIG 0x5f504d5f /* _MP_ */ -#define NEXT(X) ((X) += 4) -static int -search_for_sig(u_int32_t target, int count) -{ - int x; - u_int32_t *addr = (u_int32_t *) (KERNBASE + target); - - for (x = 0; x < count; NEXT(x)) - if (addr[x] == MP_SIG) - /* make array index a byte index */ - return (target + (x * sizeof(u_int32_t))); - - return -1; -} - - -static basetable_entry basetable_entry_types[] = -{ - {0, 20, "Processor"}, - {1, 8, "Bus"}, - {2, 8, "I/O APIC"}, - {3, 8, "I/O INT"}, - {4, 8, "Local INT"} -}; - -typedef struct BUSDATA { - u_char bus_id; - enum busTypes bus_type; -} bus_datum; - -typedef struct INTDATA { - u_char int_type; - u_short int_flags; - u_char src_bus_id; - u_char src_bus_irq; - u_char dst_apic_id; - u_char dst_apic_int; - u_char int_vector; -} io_int, local_int; - -typedef struct BUSTYPENAME { - u_char type; - char name[7]; -} bus_type_name; - -static bus_type_name bus_type_table[] = -{ - {CBUS, "CBUS"}, - {CBUSII, "CBUSII"}, - {EISA, "EISA"}, - {MCA, "MCA"}, - {UNKNOWN_BUSTYPE, "---"}, - {ISA, "ISA"}, - {MCA, "MCA"}, - {UNKNOWN_BUSTYPE, "---"}, - {UNKNOWN_BUSTYPE, "---"}, - {UNKNOWN_BUSTYPE, "---"}, - {UNKNOWN_BUSTYPE, "---"}, - {UNKNOWN_BUSTYPE, "---"}, - {PCI, "PCI"}, - {UNKNOWN_BUSTYPE, "---"}, - {UNKNOWN_BUSTYPE, "---"}, - {UNKNOWN_BUSTYPE, "---"}, - {UNKNOWN_BUSTYPE, "---"}, - {XPRESS, "XPRESS"}, - {UNKNOWN_BUSTYPE, "---"} -}; -/* from MP spec v1.4, table 5-1 */ -static int default_data[7][5] = -{ -/* nbus, id0, type0, id1, type1 */ - {1, 0, ISA, 255, 255}, - {1, 0, EISA, 255, 255}, - {1, 0, EISA, 255, 255}, - {1, 0, MCA, 255, 255}, - {2, 0, ISA, 1, PCI}, - {2, 0, EISA, 1, PCI}, - {2, 0, MCA, 1, PCI} -}; - - -/* the bus data */ -static bus_datum *bus_data; - -/* the IO INT data, one entry per possible APIC INTerrupt */ -static io_int *io_apic_ints; - -static int nintrs; - -static int processor_entry(proc_entry_ptr entry, int cpu); -static int bus_entry(bus_entry_ptr entry, int bus); -static int io_apic_entry(io_apic_entry_ptr entry, int apic); -static int int_entry(int_entry_ptr entry, int intr); -static int lookup_bus_type(char *name); - - -/* - * 1st pass on motherboard's Intel MP specification table. - * - * initializes: - * mp_ncpus = 1 - * - * determines: - * cpu_apic_address (common to all CPUs) - * io_apic_address[N] - * mp_naps - * mp_nbusses - * mp_napics - * nintrs - */ -static void -mptable_pass1(void) -{ - int x; - mpcth_t cth; - int totalSize; - void* position; - int count; - int type; - u_int id_mask; - - POSTCODE(MPTABLE_PASS1_POST); - - /* clear various tables */ - for (x = 0; x < NAPICID; ++x) { - io_apic_address[x] = ~0; /* IO APIC address table */ + if (smp_cpus == mp_ncpus) { + /* enable IPI's, tlb shootdown, freezes etc */ + atomic_store_rel_int(&smp_started, 1); + smp_active = 1; /* historic */ } - /* init everything to empty */ - mp_naps = 0; - mp_nbusses = 0; - mp_napics = 0; - nintrs = 0; - id_mask = 0; + mtx_unlock_spin(&ap_boot_mtx); - /* check for use of 'default' configuration */ - if (MPFPS_MPFB1 != 0) { - /* use default addresses */ - cpu_apic_address = DEFAULT_APIC_BASE; - io_apic_address[0] = DEFAULT_IO_APIC_BASE; + /* wait until all the AP's are up */ + while (smp_started == 0) + ia32_pause(); - /* fill in with defaults */ - mp_naps = 2; /* includes BSP */ - mp_maxid = 1; - mp_nbusses = default_data[MPFPS_MPFB1 - 1][0]; -#if defined(APIC_IO) - mp_napics = 1; - nintrs = 16; -#endif /* APIC_IO */ - } - else { - if ((cth = mpfps->pap) == 0) - panic("MP Configuration Table Header MISSING!"); + /* ok, now grab sched_lock and enter the scheduler */ + mtx_lock_spin(&sched_lock); - cpu_apic_address = (vm_offset_t) cth->apic_address; + binuptime(PCPU_PTR(switchtime)); + PCPU_SET(switchticks, ticks); - /* walk the table, recording info of interest */ - totalSize = cth->base_table_length - sizeof(struct MPCTH); - position = (u_char *) cth + sizeof(struct MPCTH); - count = cth->entry_count; + cpu_throw(NULL, choosethread()); /* doesn't return */ - while (count--) { - switch (type = *(u_char *) position) { - case 0: /* processor_entry */ - if (((proc_entry_ptr)position)->cpu_flags - & PROCENTRY_FLAG_EN) { - ++mp_naps; - mp_maxid++; - id_mask |= 1 << - ((proc_entry_ptr)position)->apic_id; - } - break; - case 1: /* bus_entry */ - ++mp_nbusses; - break; - case 2: /* io_apic_entry */ - if (((io_apic_entry_ptr)position)->apic_flags - & IOAPICENTRY_FLAG_EN) - io_apic_address[mp_napics++] = - (vm_offset_t)((io_apic_entry_ptr) - position)->apic_address; - break; - case 3: /* int_entry */ - ++nintrs; - break; - case 4: /* int_entry */ - break; - default: - panic("mpfps Base Table HOSED!"); - /* NOTREACHED */ - } - - totalSize -= basetable_entry_types[type].length; - (u_char*)position += basetable_entry_types[type].length; - } - } - - /* qualify the numbers */ - if (mp_naps > MAXCPU) { - printf("Warning: only using %d of %d available CPUs!\n", - MAXCPU, mp_naps); - mp_naps = MAXCPU; - } - - /* See if we need to fixup HT logical CPUs. */ - mptable_hyperthread_fixup(id_mask); - - /* - * Count the BSP. - * This is also used as a counter while starting the APs. - */ - mp_ncpus = 1; - - --mp_naps; /* subtract the BSP */ + panic("scheduler returned us to %s", __func__); + /* NOTREACHED */ } - -/* - * 2nd pass on motherboard's Intel MP specification table. - * - * sets: - * boot_cpu_id - * ID_TO_IO(N), phy APIC ID to log CPU/IO table - * CPU_TO_ID(N), logical CPU to APIC ID table - * IO_TO_ID(N), logical IO to APIC ID table - * bus_data[N] - * io_apic_ints[N] +/******************************************************************* + * local functions and data */ -static int -mptable_pass2(void) -{ - struct PROCENTRY proc; - int x; - mpcth_t cth; - int totalSize; - void* position; - int count; - int type; - int apic, bus, cpu, intr; - int i, j; - - POSTCODE(MPTABLE_PASS2_POST); - - /* Initialize fake proc entry for use with HT fixup. */ - bzero(&proc, sizeof(proc)); - proc.type = 0; - proc.cpu_flags = PROCENTRY_FLAG_EN; - - MALLOC(io_apic_versions, u_int32_t *, sizeof(u_int32_t) * mp_napics, - M_DEVBUF, M_WAITOK); - MALLOC(ioapic, volatile ioapic_t **, sizeof(ioapic_t *) * mp_napics, - M_DEVBUF, M_WAITOK); - MALLOC(io_apic_ints, io_int *, sizeof(io_int) * (nintrs + 1), - M_DEVBUF, M_WAITOK); - MALLOC(bus_data, bus_datum *, sizeof(bus_datum) * mp_nbusses, - M_DEVBUF, M_WAITOK); - - bzero(ioapic, sizeof(ioapic_t *) * mp_napics); - - for (i = 0; i < mp_napics; i++) { - for (j = 0; j < mp_napics; j++) { - /* same page frame as a previous IO apic? */ - if (((vm_offset_t)SMPpt[NPTEPG-2-j] & PG_FRAME) == - (io_apic_address[i] & PG_FRAME)) { - ioapic[i] = (ioapic_t *)((u_int)SMP_prvspace - + (NPTEPG-2-j) * PAGE_SIZE - + (io_apic_address[i] & PAGE_MASK)); - break; - } - /* use this slot if available */ - if (((vm_offset_t)SMPpt[NPTEPG-2-j] & PG_FRAME) == 0) { - SMPpt[NPTEPG-2-j] = (pt_entry_t)(PG_V | PG_RW | - (io_apic_address[i] & PG_FRAME)); - ioapic[i] = (ioapic_t *)((u_int)SMP_prvspace - + (NPTEPG-2-j) * PAGE_SIZE - + (io_apic_address[i] & PAGE_MASK)); - break; - } - } - } - - /* clear various tables */ - for (x = 0; x < NAPICID; ++x) { - ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */ - CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */ - IO_TO_ID(x) = -1; /* logical IO to APIC ID table */ - } - - /* clear bus data table */ - for (x = 0; x < mp_nbusses; ++x) - bus_data[x].bus_id = 0xff; - - /* clear IO APIC INT table */ - for (x = 0; x < (nintrs + 1); ++x) { - io_apic_ints[x].int_type = 0xff; - io_apic_ints[x].int_vector = 0xff; - } - - /* setup the cpu/apic mapping arrays */ - boot_cpu_id = -1; - /* record whether PIC or virtual-wire mode */ - picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0; - - /* check for use of 'default' configuration */ - if (MPFPS_MPFB1 != 0) - return MPFPS_MPFB1; /* return default configuration type */ - - if ((cth = mpfps->pap) == 0) - panic("MP Configuration Table Header MISSING!"); - - /* walk the table, recording info of interest */ - totalSize = cth->base_table_length - sizeof(struct MPCTH); - position = (u_char *) cth + sizeof(struct MPCTH); - count = cth->entry_count; - apic = bus = intr = 0; - cpu = 1; /* pre-count the BSP */ - - while (count--) { - switch (type = *(u_char *) position) { - case 0: - if (processor_entry(position, cpu)) { - if (logical_cpus != 0 && - cpu % logical_cpus != 0) - logical_cpus_mask |= (1 << cpu); - ++cpu; - } - if (need_hyperthreading_fixup) { - /* - * Create fake mptable processor entries - * and feed them to processor_entry() to - * enumerate the logical CPUs. - */ - proc.apic_id = ((proc_entry_ptr)position)->apic_id; - for (i = 1; i < logical_cpus; i++) { - proc.apic_id++; - (void)processor_entry(&proc, cpu); - logical_cpus_mask |= (1 << cpu); - cpu++; - } - } - break; - case 1: - if (bus_entry(position, bus)) - ++bus; - break; - case 2: - if (io_apic_entry(position, apic)) - ++apic; - break; - case 3: - if (int_entry(position, intr)) - ++intr; - break; - case 4: - /* int_entry(position); */ - break; - default: - panic("mpfps Base Table HOSED!"); - /* NOTREACHED */ - } - - totalSize -= basetable_entry_types[type].length; - (u_char *) position += basetable_entry_types[type].length; - } - - if (boot_cpu_id == -1) - panic("NO BSP found!"); - - /* report fact that its NOT a default configuration */ - return 0; -} - /* - * Check if we should perform a hyperthreading "fix-up" to - * enumerate any logical CPU's that aren't already listed - * in the table. + * Set the APIC logical IDs. * - * XXX: We assume that all of the physical CPUs in the - * system have the same number of logical CPUs. - * - * XXX: We assume that APIC ID's are allocated such that - * the APIC ID's for a physical processor are aligned - * with the number of logical CPU's in the processor. + * We want to cluster logical CPU's within the same APIC ID cluster. + * Since logical CPU's are aligned simply filling in the clusters in + * APIC ID order works fine. Note that this does not try to balance + * the number of CPU's in each cluster. (XXX?) */ static void -mptable_hyperthread_fixup(u_int id_mask) +set_logical_apic_ids(void) { - u_int i, id; - int logical; + u_int apic_id, cluster, cluster_id; - /* Nothing to do if there is no HTT support. */ - if ((cpu_feature & CPUID_HTT) == 0) - return; - logical_cpus = (cpu_procinfo & CPUID_HTT_CORES) >> 16; - if (logical_cpus <= 1) - return; - - /* - * For each APIC ID of a CPU that is set in the mask, - * scan the other candidate APIC ID's for this - * physical processor. If any of those ID's are - * already in the table, then kill the fixup. - */ - for (id = 0; id <= MAXCPU; id++) { - if ((id_mask & 1 << id) == 0) + /* Force us to allocate cluster 0 at the start. */ + cluster = -1; + cluster_id = APIC_MAX_INTRACLUSTER_ID; + for (apic_id = 0; apic_id < MAXCPU; apic_id++) { + if (!cpu_info[apic_id].cpu_present) continue; - /* First, make sure we are on a logical_cpus boundary. */ - if (id % logical_cpus != 0) - return; - for (i = id + 1; i < id + logical_cpus; i++) - if ((id_mask & 1 << i) != 0) - return; - } - - /* - * Ok, the ID's checked out, so enable the fixup. We have to fixup - * mp_naps and mp_maxid right now. - */ - need_hyperthreading_fixup = 1; - mp_maxid *= logical_cpus; - mp_naps *= logical_cpus; - - /* - * Now setup the cpu topology map. - */ - mp_top.ct_count = mp_naps / logical_cpus; - mp_top.ct_group = mp_groups; - - /* - * The first logical id is directly after the last valid physical id. - */ - logical = mp_top.ct_count + 1; - - for (i = 0; i < mp_top.ct_count; i++) { - int j; - - mp_groups[i].cg_mask = (1 << i); - for (j = 1; j < logical_cpus; j++) - mp_groups[i].cg_mask |= (1 << logical++); - mp_groups[i].cg_count = logical_cpus; - mp_groups[i].cg_children = 0; + if (cluster_id == APIC_MAX_INTRACLUSTER_ID) { + cluster = ioapic_next_logical_cluster(); + cluster_id = 0; + } else + cluster_id++; + /* XXXTEST */ + printf("APIC ID: physical %u, logical %u:%u\n", apic_id, + cluster, cluster_id); + lapic_set_logical_id(apic_id, cluster, cluster_id); } - - smp_topology = &mp_top; } -void -assign_apic_irq(int apic, int intpin, int irq) -{ - int x; - - if (int_to_apicintpin[irq].ioapic != -1) - panic("assign_apic_irq: inconsistent table"); - - int_to_apicintpin[irq].ioapic = apic; - int_to_apicintpin[irq].int_pin = intpin; - int_to_apicintpin[irq].apic_address = ioapic[apic]; - int_to_apicintpin[irq].redirindex = IOAPIC_REDTBL + 2 * intpin; - - for (x = 0; x < nintrs; x++) { - if ((io_apic_ints[x].int_type == 0 || - io_apic_ints[x].int_type == 3) && - io_apic_ints[x].int_vector == 0xff && - io_apic_ints[x].dst_apic_id == IO_TO_ID(apic) && - io_apic_ints[x].dst_apic_int == intpin) - io_apic_ints[x].int_vector = irq; - } -} - -void -revoke_apic_irq(int irq) -{ - int x; - int oldapic; - int oldintpin; - - if (int_to_apicintpin[irq].ioapic == -1) - panic("revoke_apic_irq: inconsistent table"); - - oldapic = int_to_apicintpin[irq].ioapic; - oldintpin = int_to_apicintpin[irq].int_pin; - - int_to_apicintpin[irq].ioapic = -1; - int_to_apicintpin[irq].int_pin = 0; - int_to_apicintpin[irq].apic_address = NULL; - int_to_apicintpin[irq].redirindex = 0; - - for (x = 0; x < nintrs; x++) { - if ((io_apic_ints[x].int_type == 0 || - io_apic_ints[x].int_type == 3) && - io_apic_ints[x].int_vector != 0xff && - io_apic_ints[x].dst_apic_id == IO_TO_ID(oldapic) && - io_apic_ints[x].dst_apic_int == oldintpin) - io_apic_ints[x].int_vector = 0xff; - } -} - - -static void -allocate_apic_irq(int intr) -{ - int apic; - int intpin; - int irq; - - if (io_apic_ints[intr].int_vector != 0xff) - return; /* Interrupt handler already assigned */ - - if (io_apic_ints[intr].int_type != 0 && - (io_apic_ints[intr].int_type != 3 || - (io_apic_ints[intr].dst_apic_id == IO_TO_ID(0) && - io_apic_ints[intr].dst_apic_int == 0))) - return; /* Not INT or ExtInt on != (0, 0) */ - - irq = 0; - while (irq < APIC_INTMAPSIZE && - int_to_apicintpin[irq].ioapic != -1) - irq++; - - if (irq >= APIC_INTMAPSIZE) - return; /* No free interrupt handlers */ - - apic = ID_TO_IO(io_apic_ints[intr].dst_apic_id); - intpin = io_apic_ints[intr].dst_apic_int; - - assign_apic_irq(apic, intpin, irq); - io_apic_setup_intpin(apic, intpin); -} - - -static void -swap_apic_id(int apic, int oldid, int newid) -{ - int x; - int oapic; - - - if (oldid == newid) - return; /* Nothing to do */ - - printf("Changing APIC ID for IO APIC #%d from %d to %d in MP table\n", - apic, oldid, newid); - - /* Swap physical APIC IDs in interrupt entries */ - for (x = 0; x < nintrs; x++) { - if (io_apic_ints[x].dst_apic_id == oldid) - io_apic_ints[x].dst_apic_id = newid; - else if (io_apic_ints[x].dst_apic_id == newid) - io_apic_ints[x].dst_apic_id = oldid; - } - - /* Swap physical APIC IDs in IO_TO_ID mappings */ - for (oapic = 0; oapic < mp_napics; oapic++) - if (IO_TO_ID(oapic) == newid) - break; - - if (oapic < mp_napics) { - printf("Changing APIC ID for IO APIC #%d from " - "%d to %d in MP table\n", - oapic, newid, oldid); - IO_TO_ID(oapic) = oldid; - } - IO_TO_ID(apic) = newid; -} - - -static void -fix_id_to_io_mapping(void) -{ - int x; - - for (x = 0; x < NAPICID; x++) - ID_TO_IO(x) = -1; - - for (x = 0; x <= mp_naps; x++) - if (CPU_TO_ID(x) < NAPICID) - ID_TO_IO(CPU_TO_ID(x)) = x; - - for (x = 0; x < mp_napics; x++) - if (IO_TO_ID(x) < NAPICID) - ID_TO_IO(IO_TO_ID(x)) = x; -} - - -static int -first_free_apic_id(void) -{ - int freeid, x; - - for (freeid = 0; freeid < NAPICID; freeid++) { - for (x = 0; x <= mp_naps; x++) - if (CPU_TO_ID(x) == freeid) - break; - if (x <= mp_naps) - continue; - for (x = 0; x < mp_napics; x++) - if (IO_TO_ID(x) == freeid) - break; - if (x < mp_napics) - continue; - return freeid; - } - return freeid; -} - - -static int -io_apic_id_acceptable(int apic, int id) -{ - int cpu; /* Logical CPU number */ - int oapic; /* Logical IO APIC number for other IO APIC */ - - if (id >= NAPICID) - return 0; /* Out of range */ - - for (cpu = 0; cpu <= mp_naps; cpu++) - if (CPU_TO_ID(cpu) == id) - return 0; /* Conflict with CPU */ - - for (oapic = 0; oapic < mp_napics && oapic < apic; oapic++) - if (IO_TO_ID(oapic) == id) - return 0; /* Conflict with other APIC */ - - return 1; /* ID is acceptable for IO APIC */ -} - - /* - * parse an Intel MP specification table - */ -static void -fix_mp_table(void) -{ - int x; - int id; - int bus_0 = 0; /* Stop GCC warning */ - int bus_pci = 0; /* Stop GCC warning */ - int num_pci_bus; - int apic; /* IO APIC unit number */ - int freeid; /* Free physical APIC ID */ - int physid; /* Current physical IO APIC ID */ - - /* - * Fix mis-numbering of the PCI bus and its INT entries if the BIOS - * did it wrong. The MP spec says that when more than 1 PCI bus - * exists the BIOS must begin with bus entries for the PCI bus and use - * actual PCI bus numbering. This implies that when only 1 PCI bus - * exists the BIOS can choose to ignore this ordering, and indeed many - * MP motherboards do ignore it. This causes a problem when the PCI - * sub-system makes requests of the MP sub-system based on PCI bus - * numbers. So here we look for the situation and renumber the - * busses and associated INTs in an effort to "make it right". - */ - - /* find bus 0, PCI bus, count the number of PCI busses */ - for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { - if (bus_data[x].bus_id == 0) { - bus_0 = x; - } - if (bus_data[x].bus_type == PCI) { - ++num_pci_bus; - bus_pci = x; - } - } - /* - * bus_0 == slot of bus with ID of 0 - * bus_pci == slot of last PCI bus encountered - */ - - /* check the 1 PCI bus case for sanity */ - /* if it is number 0 all is well */ - if (num_pci_bus == 1 && - bus_data[bus_pci].bus_id != 0) { - - /* mis-numbered, swap with whichever bus uses slot 0 */ - - /* swap the bus entry types */ - bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type; - bus_data[bus_0].bus_type = PCI; - - /* swap each relavant INTerrupt entry */ - id = bus_data[bus_pci].bus_id; - for (x = 0; x < nintrs; ++x) { - if (io_apic_ints[x].src_bus_id == id) { - io_apic_ints[x].src_bus_id = 0; - } - else if (io_apic_ints[x].src_bus_id == 0) { - io_apic_ints[x].src_bus_id = id; - } - } - } - - /* Assign IO APIC IDs. - * - * First try the existing ID. If a conflict is detected, try - * the ID in the MP table. If a conflict is still detected, find - * a free id. - * - * We cannot use the ID_TO_IO table before all conflicts has been - * resolved and the table has been corrected. - */ - for (apic = 0; apic < mp_napics; ++apic) { /* For all IO APICs */ - - /* First try to use the value set by the BIOS */ - physid = io_apic_get_id(apic); - if (io_apic_id_acceptable(apic, physid)) { - if (IO_TO_ID(apic) != physid) - swap_apic_id(apic, IO_TO_ID(apic), physid); - continue; - } - - /* Then check if the value in the MP table is acceptable */ - if (io_apic_id_acceptable(apic, IO_TO_ID(apic))) - continue; - - /* Last resort, find a free APIC ID and use it */ - freeid = first_free_apic_id(); - if (freeid >= NAPICID) - panic("No free physical APIC IDs found"); - - if (io_apic_id_acceptable(apic, freeid)) { - swap_apic_id(apic, IO_TO_ID(apic), freeid); - continue; - } - panic("Free physical APIC ID not usable"); - } - fix_id_to_io_mapping(); - - /* detect and fix broken Compaq MP table */ - if (apic_int_type(0, 0) == -1) { - printf("APIC_IO: MP table broken: 8259->APIC entry missing!\n"); - io_apic_ints[nintrs].int_type = 3; /* ExtInt */ - io_apic_ints[nintrs].int_vector = 0xff; /* Unassigned */ - /* XXX fixme, set src bus id etc, but it doesn't seem to hurt */ - io_apic_ints[nintrs].dst_apic_id = IO_TO_ID(0); - io_apic_ints[nintrs].dst_apic_int = 0; /* Pin 0 */ - nintrs++; - } -} - - -/* Assign low level interrupt handlers */ -static void -setup_apic_irq_mapping(void) -{ - int x; - int int_vector; - - /* Clear array */ - for (x = 0; x < APIC_INTMAPSIZE; x++) { - int_to_apicintpin[x].ioapic = -1; - int_to_apicintpin[x].int_pin = 0; - int_to_apicintpin[x].apic_address = NULL; - int_to_apicintpin[x].redirindex = 0; - } - - /* First assign ISA/EISA interrupts */ - for (x = 0; x < nintrs; x++) { - int_vector = io_apic_ints[x].src_bus_irq; - if (int_vector < APIC_INTMAPSIZE && - io_apic_ints[x].int_vector == 0xff && - int_to_apicintpin[int_vector].ioapic == -1 && - (apic_int_is_bus_type(x, ISA) || - apic_int_is_bus_type(x, EISA)) && - io_apic_ints[x].int_type == 0) { - assign_apic_irq(ID_TO_IO(io_apic_ints[x].dst_apic_id), - io_apic_ints[x].dst_apic_int, - int_vector); - } - } - - /* Assign ExtInt entry if no ISA/EISA interrupt 0 entry */ - for (x = 0; x < nintrs; x++) { - if (io_apic_ints[x].dst_apic_int == 0 && - io_apic_ints[x].dst_apic_id == IO_TO_ID(0) && - io_apic_ints[x].int_vector == 0xff && - int_to_apicintpin[0].ioapic == -1 && - io_apic_ints[x].int_type == 3) { - assign_apic_irq(0, 0, 0); - break; - } - } - /* PCI interrupt assignment is deferred */ -} - - -static int -processor_entry(proc_entry_ptr entry, int cpu) -{ - /* check for usability */ - if (!(entry->cpu_flags & PROCENTRY_FLAG_EN)) - return 0; - - if(entry->apic_id >= NAPICID) - panic("CPU APIC ID out of range (0..%d)", NAPICID - 1); - /* check for BSP flag */ - if (entry->cpu_flags & PROCENTRY_FLAG_BP) { - boot_cpu_id = entry->apic_id; - CPU_TO_ID(0) = entry->apic_id; - ID_TO_CPU(entry->apic_id) = 0; - return 0; /* its already been counted */ - } - - /* add another AP to list, if less than max number of CPUs */ - else if (cpu < MAXCPU) { - CPU_TO_ID(cpu) = entry->apic_id; - ID_TO_CPU(entry->apic_id) = cpu; - return 1; - } - - return 0; -} - - -static int -bus_entry(bus_entry_ptr entry, int bus) -{ - int x; - char c, name[8]; - - /* encode the name into an index */ - for (x = 0; x < 6; ++x) { - if ((c = entry->bus_type[x]) == ' ') - break; - name[x] = c; - } - name[x] = '\0'; - - if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) - panic("unknown bus type: '%s'", name); - - bus_data[bus].bus_id = entry->bus_id; - bus_data[bus].bus_type = x; - - return 1; -} - - -static int -io_apic_entry(io_apic_entry_ptr entry, int apic) -{ - if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) - return 0; - - IO_TO_ID(apic) = entry->apic_id; - if (entry->apic_id < NAPICID) - ID_TO_IO(entry->apic_id) = apic; - - return 1; -} - - -static int -lookup_bus_type(char *name) -{ - int x; - - for (x = 0; x < MAX_BUSTYPE; ++x) - if (strcmp(bus_type_table[x].name, name) == 0) - return bus_type_table[x].type; - - return UNKNOWN_BUSTYPE; -} - - -static int -int_entry(int_entry_ptr entry, int intr) -{ - int apic; - - io_apic_ints[intr].int_type = entry->int_type; - io_apic_ints[intr].int_flags = entry->int_flags; - io_apic_ints[intr].src_bus_id = entry->src_bus_id; - io_apic_ints[intr].src_bus_irq = entry->src_bus_irq; - if (entry->dst_apic_id == 255) { - /* This signal goes to all IO APICS. Select an IO APIC - with sufficient number of interrupt pins */ - for (apic = 0; apic < mp_napics; apic++) - if (((io_apic_read(apic, IOAPIC_VER) & - IOART_VER_MAXREDIR) >> MAXREDIRSHIFT) >= - entry->dst_apic_int) - break; - if (apic < mp_napics) - io_apic_ints[intr].dst_apic_id = IO_TO_ID(apic); - else - io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; - } else - io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; - io_apic_ints[intr].dst_apic_int = entry->dst_apic_int; - - return 1; -} - - -static int -apic_int_is_bus_type(int intr, int bus_type) -{ - int bus; - - for (bus = 0; bus < mp_nbusses; ++bus) - if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) - && ((int) bus_data[bus].bus_type == bus_type)) - return 1; - - return 0; -} - - -/* - * Given a traditional ISA INT mask, return an APIC mask. - */ -u_int -isa_apic_mask(u_int isa_mask) -{ - int isa_irq; - int apic_pin; - -#if defined(SKIP_IRQ15_REDIRECT) - if (isa_mask == (1 << 15)) { - printf("skipping ISA IRQ15 redirect\n"); - return isa_mask; - } -#endif /* SKIP_IRQ15_REDIRECT */ - - isa_irq = ffs(isa_mask); /* find its bit position */ - if (isa_irq == 0) /* doesn't exist */ - return 0; - --isa_irq; /* make it zero based */ - - apic_pin = isa_apic_irq(isa_irq); /* look for APIC connection */ - if (apic_pin == -1) - return 0; - - return (1 << apic_pin); /* convert pin# to a mask */ -} - - -/* - * Determine which APIC pin an ISA/EISA INT is attached to. - */ -#define INTTYPE(I) (io_apic_ints[(I)].int_type) -#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) -#define INTIRQ(I) (io_apic_ints[(I)].int_vector) -#define INTAPIC(I) (ID_TO_IO(io_apic_ints[(I)].dst_apic_id)) - -#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) -int -isa_apic_irq(int isa_irq) -{ - int intr; - - for (intr = 0; intr < nintrs; ++intr) { /* check each record */ - if (INTTYPE(intr) == 0) { /* standard INT */ - if (SRCBUSIRQ(intr) == isa_irq) { - if (apic_int_is_bus_type(intr, ISA) || - apic_int_is_bus_type(intr, EISA)) { - if (INTIRQ(intr) == 0xff) - return -1; /* unassigned */ - return INTIRQ(intr); /* found */ - } - } - } - } - return -1; /* NOT found */ -} - - -/* - * Determine which APIC pin a PCI INT is attached to. - */ -#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) -#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) -#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) -int -pci_apic_irq(int pciBus, int pciDevice, int pciInt) -{ - int intr; - - --pciInt; /* zero based */ - - for (intr = 0; intr < nintrs; ++intr) /* check each record */ - if ((INTTYPE(intr) == 0) /* standard INT */ - && (SRCBUSID(intr) == pciBus) - && (SRCBUSDEVICE(intr) == pciDevice) - && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ - if (apic_int_is_bus_type(intr, PCI)) { - if (INTIRQ(intr) == 0xff) - allocate_apic_irq(intr); - if (INTIRQ(intr) == 0xff) - return -1; /* unassigned */ - return INTIRQ(intr); /* exact match */ - } - - return -1; /* NOT found */ -} - -int -next_apic_irq(int irq) -{ - int intr, ointr; - int bus, bustype; - - bus = 0; - bustype = 0; - for (intr = 0; intr < nintrs; intr++) { - if (INTIRQ(intr) != irq || INTTYPE(intr) != 0) - continue; - bus = SRCBUSID(intr); - bustype = apic_bus_type(bus); - if (bustype != ISA && - bustype != EISA && - bustype != PCI) - continue; - break; - } - if (intr >= nintrs) { - return -1; - } - for (ointr = intr + 1; ointr < nintrs; ointr++) { - if (INTTYPE(ointr) != 0) - continue; - if (bus != SRCBUSID(ointr)) - continue; - if (bustype == PCI) { - if (SRCBUSDEVICE(intr) != SRCBUSDEVICE(ointr)) - continue; - if (SRCBUSLINE(intr) != SRCBUSLINE(ointr)) - continue; - } - if (bustype == ISA || bustype == EISA) { - if (SRCBUSIRQ(intr) != SRCBUSIRQ(ointr)) - continue; - } - if (INTPIN(intr) == INTPIN(ointr)) - continue; - break; - } - if (ointr >= nintrs) { - return -1; - } - return INTIRQ(ointr); -} -#undef SRCBUSLINE -#undef SRCBUSDEVICE -#undef SRCBUSID -#undef SRCBUSIRQ - -#undef INTPIN -#undef INTIRQ -#undef INTAPIC -#undef INTTYPE - - -/* - * Reprogram the MB chipset to NOT redirect an ISA INTerrupt. - * - * XXX FIXME: - * Exactly what this means is unclear at this point. It is a solution - * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard - * could route any of the ISA INTs to upper (>15) IRQ values. But most would - * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an - * option. - */ -int -undirect_isa_irq(int rirq) -{ -#if defined(READY) - if (bootverbose) - printf("Freeing redirected ISA irq %d.\n", rirq); - /** FIXME: tickle the MB redirector chip */ - return -1; -#else - if (bootverbose) - printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq); - return 0; -#endif /* READY */ -} - - -/* - * Reprogram the MB chipset to NOT redirect a PCI INTerrupt - */ -int -undirect_pci_irq(int rirq) -{ -#if defined(READY) - if (bootverbose) - printf("Freeing redirected PCI irq %d.\n", rirq); - - /** FIXME: tickle the MB redirector chip */ - return -1; -#else - if (bootverbose) - printf("Freeing (NOT implemented) redirected PCI irq %d.\n", - rirq); - return 0; -#endif /* READY */ -} - - -/* - * given a bus ID, return: - * the bus type if found - * -1 if NOT found - */ -int -apic_bus_type(int id) -{ - int x; - - for (x = 0; x < mp_nbusses; ++x) - if (bus_data[x].bus_id == id) - return bus_data[x].bus_type; - - return -1; -} - - -/* - * given a LOGICAL APIC# and pin#, return: - * the associated src bus ID if found - * -1 if NOT found - */ -int -apic_src_bus_id(int apic, int pin) -{ - int x; - - /* search each of the possible INTerrupt sources */ - for (x = 0; x < nintrs; ++x) - if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && - (pin == io_apic_ints[x].dst_apic_int)) - return (io_apic_ints[x].src_bus_id); - - return -1; /* NOT found */ -} - - -/* - * given a LOGICAL APIC# and pin#, return: - * the associated src bus IRQ if found - * -1 if NOT found - */ -int -apic_src_bus_irq(int apic, int pin) -{ - int x; - - for (x = 0; x < nintrs; x++) - if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && - (pin == io_apic_ints[x].dst_apic_int)) - return (io_apic_ints[x].src_bus_irq); - - return -1; /* NOT found */ -} - - -/* - * given a LOGICAL APIC# and pin#, return: - * the associated INTerrupt type if found - * -1 if NOT found - */ -int -apic_int_type(int apic, int pin) -{ - int x; - - /* search each of the possible INTerrupt sources */ - for (x = 0; x < nintrs; ++x) - if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && - (pin == io_apic_ints[x].dst_apic_int)) - return (io_apic_ints[x].int_type); - - return -1; /* NOT found */ -} - -int -apic_irq(int apic, int pin) -{ - int x; - int res; - - for (x = 0; x < nintrs; ++x) - if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && - (pin == io_apic_ints[x].dst_apic_int)) { - res = io_apic_ints[x].int_vector; - if (res == 0xff) - return -1; - if (apic != int_to_apicintpin[res].ioapic) - panic("apic_irq: inconsistent table"); - if (pin != int_to_apicintpin[res].int_pin) - panic("apic_irq inconsistent table (2)"); - return res; - } - return -1; -} - - -/* - * given a LOGICAL APIC# and pin#, return: - * the associated trigger mode if found - * -1 if NOT found - */ -int -apic_trigger(int apic, int pin) -{ - int x; - - /* search each of the possible INTerrupt sources */ - for (x = 0; x < nintrs; ++x) - if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && - (pin == io_apic_ints[x].dst_apic_int)) - return ((io_apic_ints[x].int_flags >> 2) & 0x03); - - return -1; /* NOT found */ -} - - -/* - * given a LOGICAL APIC# and pin#, return: - * the associated 'active' level if found - * -1 if NOT found - */ -int -apic_polarity(int apic, int pin) -{ - int x; - - /* search each of the possible INTerrupt sources */ - for (x = 0; x < nintrs; ++x) - if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && - (pin == io_apic_ints[x].dst_apic_int)) - return (io_apic_ints[x].int_flags & 0x03); - - return -1; /* NOT found */ -} - - -/* - * set data according to MP defaults - * FIXME: probably not complete yet... - */ -static void -default_mp_table(int type) -{ - int ap_cpu_id; -#if defined(APIC_IO) - int io_apic_id; - int pin; -#endif /* APIC_IO */ - -#if 0 - printf(" MP default config type: %d\n", type); - switch (type) { - case 1: - printf(" bus: ISA, APIC: 82489DX\n"); - break; - case 2: - printf(" bus: EISA, APIC: 82489DX\n"); - break; - case 3: - printf(" bus: EISA, APIC: 82489DX\n"); - break; - case 4: - printf(" bus: MCA, APIC: 82489DX\n"); - break; - case 5: - printf(" bus: ISA+PCI, APIC: Integrated\n"); - break; - case 6: - printf(" bus: EISA+PCI, APIC: Integrated\n"); - break; - case 7: - printf(" bus: MCA+PCI, APIC: Integrated\n"); - break; - default: - printf(" future type\n"); - break; - /* NOTREACHED */ - } -#endif /* 0 */ - - boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24; - ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; - - /* BSP */ - CPU_TO_ID(0) = boot_cpu_id; - ID_TO_CPU(boot_cpu_id) = 0; - - /* one and only AP */ - CPU_TO_ID(1) = ap_cpu_id; - ID_TO_CPU(ap_cpu_id) = 1; - -#if defined(APIC_IO) - /* one and only IO APIC */ - io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; - - /* - * sanity check, refer to MP spec section 3.6.6, last paragraph - * necessary as some hardware isn't properly setting up the IO APIC - */ -#if defined(REALLY_ANAL_IOAPICID_VALUE) - if (io_apic_id != 2) { -#else - if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { -#endif /* REALLY_ANAL_IOAPICID_VALUE */ - io_apic_set_id(0, 2); - io_apic_id = 2; - } - IO_TO_ID(0) = io_apic_id; - ID_TO_IO(io_apic_id) = 0; -#endif /* APIC_IO */ - - /* fill out bus entries */ - switch (type) { - case 1: - case 2: - case 3: - case 4: - case 5: - case 6: - case 7: - bus_data[0].bus_id = default_data[type - 1][1]; - bus_data[0].bus_type = default_data[type - 1][2]; - bus_data[1].bus_id = default_data[type - 1][3]; - bus_data[1].bus_type = default_data[type - 1][4]; - break; - - /* case 4: case 7: MCA NOT supported */ - default: /* illegal/reserved */ - panic("BAD default MP config: %d", type); - /* NOTREACHED */ - } - -#if defined(APIC_IO) - /* general cases from MP v1.4, table 5-2 */ - for (pin = 0; pin < 16; ++pin) { - io_apic_ints[pin].int_type = 0; - io_apic_ints[pin].int_flags = 0x05; /* edge/active-hi */ - io_apic_ints[pin].src_bus_id = 0; - io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 caught below */ - io_apic_ints[pin].dst_apic_id = io_apic_id; - io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 */ - } - - /* special cases from MP v1.4, table 5-2 */ - if (type == 2) { - io_apic_ints[2].int_type = 0xff; /* N/C */ - io_apic_ints[13].int_type = 0xff; /* N/C */ -#if !defined(APIC_MIXED_MODE) - /** FIXME: ??? */ - panic("sorry, can't support type 2 default yet"); -#endif /* APIC_MIXED_MODE */ - } - else - io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ - - if (type == 7) - io_apic_ints[0].int_type = 0xff; /* N/C */ - else - io_apic_ints[0].int_type = 3; /* vectored 8259 */ -#endif /* APIC_IO */ -} - - -/* * start each AP in our list */ static int start_all_aps(u_int boot_addr) { - int x, i, pg; #ifndef PC98 - u_char mpbiosreason; + u_char mpbiosreason; #endif - u_long mpbioswarmvec; + u_long mpbioswarmvec; struct pcpu *pc; char *stack; uintptr_t kptbase; + int i, pg, apic_id, cpu; POSTCODE(START_ALL_APS_POST); mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); - /* initialize BSP's local APIC */ - apic_initialize(); - bsp_apic_ready = 1; - /* install the AP 1st level boot code */ install_ap_tramp(boot_addr); - /* save the current value of the warm-start vector */ mpbioswarmvec = *((u_long *) WARMBOOT_OFF); #ifndef PC98 @@ -2084,18 +598,23 @@ /* set up temporary P==V mapping for AP boot */ /* XXX this is a hack, we should boot the AP on its own stack/PTD */ kptbase = (uintptr_t)(void *)KPTphys; - for (x = 0; x < NKPT; x++) - PTD[x] = (pd_entry_t)(PG_V | PG_RW | - ((kptbase + x * PAGE_SIZE) & PG_FRAME)); + for (i = 0; i < NKPT; i++) + PTD[i] = (pd_entry_t)(PG_V | PG_RW | + ((kptbase + i * PAGE_SIZE) & PG_FRAME)); invltlb(); /* start each AP */ - for (x = 1; x <= mp_naps; ++x) { + for (cpu = 0, apic_id = 0; apic_id < MAXCPU; apic_id++) { + if (!cpu_info[apic_id].cpu_present || + cpu_info[apic_id].cpu_bsp) + continue; + cpu++; - /* This is a bit verbose, it will go away soon. */ + /* save APIC ID for this logical ID */ + cpu_apic_ids[cpu] = apic_id; /* first page of AP's private space */ - pg = x * i386_btop(sizeof(struct privatespace)); + pg = cpu * i386_btop(sizeof(struct privatespace)); /* allocate a new private data page */ pc = (struct pcpu *)kmem_alloc(kernel_map, PAGE_SIZE); @@ -2110,7 +629,8 @@ (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); /* prime data page for it to use */ - pcpu_init(pc, x, sizeof(struct pcpu)); + pcpu_init(pc, cpu, sizeof(struct pcpu)); + pc->pc_apic_id = apic_id; /* setup a vector to our boot code */ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; @@ -2120,13 +640,14 @@ outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ #endif - bootSTK = &SMP_prvspace[x].idlekstack[KSTACK_PAGES * PAGE_SIZE]; - bootAP = x; + bootSTK = &SMP_prvspace[cpu].idlekstack[KSTACK_PAGES * + PAGE_SIZE]; + bootAP = cpu; /* attempt to start the Application Processor */ CHECK_INIT(99); /* setup checkpoints */ - if (!start_ap(x, boot_addr)) { - printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); + if (!start_ap(apic_id, boot_addr)) { + printf("AP #%d (PHY# %d) failed!\n", cpu, apic_id); CHECK_PRINT("trace"); /* show checkpoints */ /* better panic as the AP may be running loose */ printf("panic y/n? [y] "); @@ -2135,18 +656,12 @@ } CHECK_PRINT("trace"); /* show checkpoints */ - /* record its version info */ - cpu_apic_versions[x] = cpu_apic_versions[0]; - - all_cpus |= (1 << x); /* record AP in CPU map */ + all_cpus |= (1 << cpu); /* record AP in CPU map */ } /* build our map of 'other' CPUs */ PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); - /* fill in our (BSP) APIC version */ - cpu_apic_versions[0] = lapic.version; - /* restore the warmstart vector */ *(u_long *) WARMBOOT_OFF = mpbioswarmvec; #ifndef PC98 @@ -2167,11 +682,12 @@ SMPpt[1 + i] = (pt_entry_t) (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); - for (x = 0; x < NKPT; x++) - PTD[x] = 0; + for (i = 0; i < NKPT; i++) + PTD[i] = 0; + pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1); /* number of APs actually started */ - return mp_ncpus - 1; + return mp_naps; } /* @@ -2200,6 +716,7 @@ POSTCODE(INSTALL_AP_TRAMP_POST); + pmap_kenter(boot_addr + KERNBASE, boot_addr); for (x = 0; x < size; ++x) *dst++ = *src++; @@ -2241,23 +758,18 @@ * but it seems to work. */ static int -start_ap(int logical_cpu, u_int boot_addr) +start_ap(int apic_id, u_int boot_addr) { - int physical_cpu; - int vector; - int cpus; - u_long icr_lo, icr_hi; + int vector, ms; + int cpus; POSTCODE(START_AP_POST); - /* get the PHYSICAL APIC ID# */ - physical_cpu = CPU_TO_ID(logical_cpu); - /* calculate the vector */ vector = (boot_addr >> 12) & 0xff; /* used as a watchpoint to signal AP startup */ - cpus = mp_ncpus; + cpus = mp_naps; /* * first we do an INIT/RESET IPI this INIT IPI might be run, reseting @@ -2266,31 +778,20 @@ * ignored. */ - /* setup the address for the target AP */ - icr_hi = lapic.icr_hi & ~APIC_ID_MASK; - icr_hi |= (physical_cpu << 24); - lapic.icr_hi = icr_hi; - - /* setup common fields for subsequent IPIs */ - icr_lo = lapic.icr_lo & APIC_ICRLO_RESV_MASK; - icr_lo |= APIC_DESTMODE_PHY; - /* do an INIT IPI: assert RESET */ - lapic.icr_lo = icr_lo | APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | - APIC_LEVEL_ASSERT | APIC_DELMODE_INIT; + lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | + APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id); /* wait for pending status end */ - while (lapic.icr_lo & APIC_DELSTAT_MASK) - /* spin */ ; + lapic_ipi_wait(-1); /* do an INIT IPI: deassert RESET */ - lapic.icr_lo = icr_lo | APIC_DEST_ALLESELF | APIC_TRIGMOD_LEVEL | - APIC_LEVEL_DEASSERT | APIC_DELMODE_INIT; + lapic_ipi_raw(APIC_DEST_ALLESELF | APIC_TRIGMOD_LEVEL | + APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, 0); /* wait for pending status end */ - u_sleep(10000); /* wait ~10mS */ - while (lapic.icr_lo & APIC_DELSTAT_MASK) - /* spin */ ; + DELAY(10000); /* wait ~10mS */ + lapic_ipi_wait(-1); /* * next we do a STARTUP IPI: the previous INIT IPI might still be @@ -2302,11 +803,11 @@ */ /* do a STARTUP IPI */ - lapic.icr_lo = icr_lo | APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | - APIC_LEVEL_DEASSERT | APIC_DELMODE_STARTUP | vector; - while (lapic.icr_lo & APIC_DELSTAT_MASK) - /* spin */ ; - u_sleep(200); /* wait ~200uS */ + lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | + APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP | + vector, apic_id); + lapic_ipi_wait(-1); + DELAY(200); /* wait ~200uS */ /* * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF @@ -2315,23 +816,21 @@ * recognized after hardware RESET or INIT IPI. */ - lapic.icr_lo = icr_lo | APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | - APIC_LEVEL_DEASSERT | APIC_DELMODE_STARTUP | vector; - while (lapic.icr_lo & APIC_DELSTAT_MASK) - /* spin */ ; - u_sleep(200); /* wait ~200uS */ + lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | + APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP | + vector, apic_id); + lapic_ipi_wait(-1); + DELAY(200); /* wait ~200uS */ - /* wait for it to start */ - set_apic_timer(5000000);/* == 5 seconds */ - while (read_apic_timer()) - if (mp_ncpus > cpus) + /* Wait up to 5 seconds for it to start. */ + for (ms = 0; ms < 5000; ms++) { + if (mp_naps > cpus) return 1; /* return SUCCESS */ - + DELAY(1000); + } return 0; /* return FAILURE */ } -#if defined(APIC_IO) - #ifdef COUNT_XINVLTLB_HITS u_int xhits_gbl[MAXCPU]; u_int xhits_pg[MAXCPU]; @@ -2366,7 +865,7 @@ &ipi_masked_range, 0, ""); SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW, &ipi_masked_range_size, 0, ""); -#endif +#endif /* COUNT_XINVLTLB_HITS */ /* * Flush the TLB on all other CPU's @@ -2375,22 +874,17 @@ smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2) { u_int ncpu; - register_t eflags; ncpu = mp_ncpus - 1; /* does not shootdown self */ if (ncpu < 1) return; /* no other cpus */ - eflags = read_eflags(); - if ((eflags & PSL_I) == 0) - panic("absolutely cannot call smp_ipi_shootdown with interrupts already disabled"); - mtx_lock_spin(&smp_tlb_mtx); + mtx_assert(&smp_tlb_mtx, MA_OWNED); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); ipi_all_but_self(vector); while (smp_tlb_wait < ncpu) ia32_pause(); - mtx_unlock_spin(&smp_tlb_mtx); } /* @@ -2443,7 +937,6 @@ smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) { int ncpu, othercpus; - register_t eflags; othercpus = mp_ncpus - 1; if (mask == (u_int)-1) { @@ -2465,10 +958,7 @@ if (ncpu < 1) return; } - eflags = read_eflags(); - if ((eflags & PSL_I) == 0) - panic("absolutely cannot call smp_targeted_ipi_shootdown with interrupts already disabled"); - mtx_lock_spin(&smp_tlb_mtx); + mtx_assert(&smp_tlb_mtx, MA_OWNED); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); @@ -2478,40 +968,33 @@ ipi_selected(mask, vector); while (smp_tlb_wait < ncpu) ia32_pause(); - mtx_unlock_spin(&smp_tlb_mtx); } -#endif void smp_invltlb(void) { -#if defined(APIC_IO) if (smp_started) { smp_tlb_shootdown(IPI_INVLTLB, 0, 0); #ifdef COUNT_XINVLTLB_HITS ipi_global++; #endif } -#endif /* APIC_IO */ } void smp_invlpg(vm_offset_t addr) { -#if defined(APIC_IO) if (smp_started) { smp_tlb_shootdown(IPI_INVLPG, addr, 0); #ifdef COUNT_XINVLTLB_HITS ipi_page++; #endif } -#endif /* APIC_IO */ } void smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2) { -#if defined(APIC_IO) if (smp_started) { smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); #ifdef COUNT_XINVLTLB_HITS @@ -2519,39 +1002,33 @@ ipi_range_size += (addr2 - addr1) / PAGE_SIZE; #endif } -#endif /* APIC_IO */ } void smp_masked_invltlb(u_int mask) { -#if defined(APIC_IO) if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); #ifdef COUNT_XINVLTLB_HITS ipi_masked_global++; #endif } -#endif /* APIC_IO */ } void smp_masked_invlpg(u_int mask, vm_offset_t addr) { -#if defined(APIC_IO) if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); #ifdef COUNT_XINVLTLB_HITS ipi_masked_page++; #endif } -#endif /* APIC_IO */ } void smp_masked_invlpg_range(u_int mask, vm_offset_t addr1, vm_offset_t addr2) { -#if defined(APIC_IO) if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); #ifdef COUNT_XINVLTLB_HITS @@ -2559,101 +1036,18 @@ ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE; #endif } -#endif /* APIC_IO */ } /* - * This is called once the rest of the system is up and running and we're - * ready to let the AP's out of the pen. - */ -void -ap_init(void) -{ - u_int apic_id; - - /* spin until all the AP's are ready */ - while (!aps_ready) - ia32_pause(); - - /* BSP may have changed PTD while we were waiting */ - invltlb(); - -#if defined(I586_CPU) && !defined(NO_F00F_HACK) - lidt(&r_idt); -#endif - - /* set up CPU registers and state */ - cpu_setregs(); - - /* set up FPU state on the AP */ - npxinit(__INITIAL_NPXCW__); - - /* set up SSE registers */ - enable_sse(); - - /* A quick check from sanity claus */ - apic_id = (apic_id_to_logical[(lapic.id & 0x0f000000) >> 24]); - if (PCPU_GET(cpuid) != apic_id) { - printf("SMP: cpuid = %d\n", PCPU_GET(cpuid)); - printf("SMP: apic_id = %d\n", apic_id); - printf("PTD[MPPTDI] = %#jx\n", (uintmax_t)PTD[MPPTDI]); - panic("cpuid mismatch! boom!!"); - } - - /* Init local apic for irq's */ - apic_initialize(); - - /* Set memory range attributes for this CPU to match the BSP */ - mem_range_AP_init(); - - mtx_lock_spin(&ap_boot_mtx); - - smp_cpus++; - - CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); - printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); - - /* Build our map of 'other' CPUs. */ - PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); - - if (bootverbose) - apic_dump("ap_init()"); - - if (smp_cpus == mp_ncpus) { - /* enable IPI's, tlb shootdown, freezes etc */ - atomic_store_rel_int(&smp_started, 1); - smp_active = 1; /* historic */ - } - - mtx_unlock_spin(&ap_boot_mtx); - - /* wait until all the AP's are up */ - while (smp_started == 0) - ia32_pause(); - - /* ok, now grab sched_lock and enter the scheduler */ - mtx_lock_spin(&sched_lock); - - binuptime(PCPU_PTR(switchtime)); - PCPU_SET(switchticks, ticks); - - cpu_throw(NULL, choosethread()); /* doesn't return */ - - panic("scheduler returned us to %s", __func__); -} - -/* * For statclock, we send an IPI to all CPU's to have them call this * function. - * - * WARNING! unpend() will call statclock() directly and skip this - * routine. */ void forwarded_statclock(struct clockframe frame) { + CTR0(KTR_SMP, "forwarded_statclock"); if (profprocs != 0) profclock(&frame); if (pscnt == psdiv) @@ -2681,14 +1075,12 @@ * sched_lock if we could simply peek at the CPU to determine the user/kernel * state and call hardclock_process() on the CPU receiving the clock interrupt * and then just use a simple IPI to handle any ast's if needed. - * - * WARNING! unpend() will call hardclock_process() directly and skip this - * routine. */ void forwarded_hardclock(struct clockframe frame) { + CTR0(KTR_SMP, "forwarded_hardclock"); hardclock_process(&frame); } @@ -2707,31 +1099,22 @@ ipi_selected(map, IPI_HARDCLOCK); } -#ifdef APIC_INTR_REORDER -/* - * Maintain mapping from softintr vector to isr bit in local apic. - */ -void -set_lapic_isrloc(int intr, int vector) -{ - if (intr < 0 || intr > 32) - panic("set_apic_isrloc: bad intr argument: %d",intr); - if (vector < ICU_OFFSET || vector > 255) - panic("set_apic_isrloc: bad vector argument: %d",vector); - apic_isrbit_location[intr].location = &lapic.isr0 + ((vector>>5)<<2); - apic_isrbit_location[intr].bit = (1<<(vector & 31)); -} -#endif - /* * send an IPI to a set of cpus. */ void ipi_selected(u_int32_t cpus, u_int ipi) { + int cpu; CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi); - selected_apic_ipi(cpus, ipi, APIC_DELMODE_FIXED); + while ((cpu = ffs(cpus)) != 0) { + cpu--; + KASSERT(cpu_apic_ids[cpu] != -1, + ("IPI to non-existent CPU %d", cpu)); + lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]); + cpus &= ~(1 << cpu); + } } /* @@ -2742,7 +1125,7 @@ { CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); - apic_ipi(APIC_DEST_ALLISELF, ipi, APIC_DELMODE_FIXED); + lapic_ipi_vectored(ipi, APIC_IPI_DEST_ALL); } /* @@ -2753,7 +1136,7 @@ { CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); - apic_ipi(APIC_DEST_ALLESELF, ipi, APIC_DELMODE_FIXED); + lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); } /* @@ -2764,9 +1147,13 @@ { CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); - apic_ipi(APIC_DEST_SELF, ipi, APIC_DELMODE_FIXED); + lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF); } +/* + * This is called once the rest of the system is up and running and we're + * ready to let the AP's out of the pen. + */ static void release_aps(void *dummy __unused) { Index: i386/i386/mpboot.s =========================================================================== --- sys/i386/i386/mpboot.s 2003/10/30 14:48:07 #4 +++ i386/i386/mpboot.s 2003/10/30 14:48:07 @@ -37,7 +37,7 @@ #include "opt_pmap.h" #include /* miscellaneous asm macros */ -#include +#include #include #include "assym.s" @@ -77,6 +77,32 @@ NON_GPROF_ENTRY(MPentry) CHECKPOINT(0x36, 3) + /* + * Enable features on this processor. We don't support SMP on + * CPUs older than a Pentium, so we know that we can use the cpuid + * instruction. + */ + movl $1,%eax + cpuid /* Retrieve features */ + movl %cr4,%eax +#ifndef DISABLE_PSE + testl $CPUID_PSE,%edx + jz 1f + orl $CR4_PSE,%eax /* Enable PSE */ +1: +#endif +#ifndef DISABLE_PG_G + testl $CPUID_PGE,%edx + jz 1f + orl $CR4_PGE,%eax /* Enable PGE */ +1: +#endif + testl $CPUID_VME,%edx + jz 1f + orl $CR4_VME,%eax /* Enable VME */ +1: + movl %eax,%cr4 + /* Now enable paging mode */ #ifdef PAE movl R(IdlePDPT), %eax @@ -88,22 +114,6 @@ movl R(IdlePTD), %eax movl %eax,%cr3 #endif -#ifndef DISABLE_PSE - cmpl $0, R(pseflag) - je 1f - movl %cr4, %eax - orl $CR4_PSE, %eax - movl %eax, %cr4 -1: -#endif -#ifndef DISABLE_PG_G - cmpl $0, R(pgeflag) - je 2f - movl %cr4, %eax - orl $CR4_PGE, %eax - movl %eax, %cr4 -2: -#endif movl %cr0,%eax orl $CR0_PE|CR0_PG,%eax /* enable paging */ movl %eax,%cr0 /* let the games begin! */ @@ -118,32 +128,6 @@ mp_begin: /* now running relocated at KERNBASE */ CHECKPOINT(0x37, 4) call init_secondary /* load i386 tables */ - CHECKPOINT(0x38, 5) - - /* - * If the [BSP] CPU has support for VME, turn it on. - */ - testl $CPUID_VME, cpu_feature /* XXX WRONG! BSP! */ - jz 1f - movl %cr4, %eax - orl $CR4_VME, %eax - movl %eax, %cr4 -1: - - /* disable the APIC, just to be SURE */ - movl lapic+LA_SVR, %eax /* get spurious vector reg. */ - andl $~APIC_SVR_SWEN, %eax /* clear software enable bit */ - movl %eax, lapic+LA_SVR - - /* signal our startup to the BSP */ - movl lapic+LA_VER, %eax /* our version reg contents */ - movl %eax, cpu_apic_versions /* into [ 0 ] */ - incl mp_ncpus /* signal BSP */ - - CHECKPOINT(0x39, 6) - - /* Now, let's prepare for some REAL WORK :-) This doesn't return. */ - call ap_init /* * This is the embedded trampoline or bootstrap that is Index: i386/i386/mptable.c =========================================================================== *** /dev/null Thu Oct 30 14:47:44 2003 --- i386/i386/mptable.c Thu Oct 30 14:48:09 2003 *************** *** 0 **** --- 1,949 ---- + /*- + * Copyright (c) 2003 John Baldwin + * Copyright (c) 1996, by Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD: src/sys/i386/i386/mp_machdep.c,v 1.208 2003/05/15 05:12:24 alc Exp $ + */ + + #include + #include + #include + #include + #include + + #include + #include + #include + + #include + #include + #include + #include + #include + #include + #include + + #include + + /* EISA Edge/Level trigger control registers */ + #define ELCR0 0x4d0 /* eisa irq 0-7 */ + #define ELCR1 0x4d1 /* eisa irq 8-15 */ + + /* string defined by the Intel MP Spec as identifying the MP table */ + #define MP_SIG 0x5f504d5f /* _MP_ */ + + #define NAPICID 16 /* Max number of I/O APIC's */ + + #ifdef PC98 + #define BIOS_BASE (0xe8000) + #define BIOS_SIZE (0x18000) + #else + #define BIOS_BASE (0xf0000) + #define BIOS_SIZE (0x10000) + #endif + #define BIOS_COUNT (BIOS_SIZE/4) + + typedef void mptable_entry_handler(u_char *entry, void *arg); + + static basetable_entry basetable_entry_types[] = + { + {0, 20, "Processor"}, + {1, 8, "Bus"}, + {2, 8, "I/O APIC"}, + {3, 8, "I/O INT"}, + {4, 8, "Local INT"} + }; + + typedef struct BUSDATA { + u_char bus_id; + enum busTypes bus_type; + } bus_datum; + + typedef struct INTDATA { + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; + u_char int_vector; + } io_int, local_int; + + typedef struct BUSTYPENAME { + u_char type; + char name[7]; + } bus_type_name; + + /* From MP spec v1.4, table 4-8. */ + static bus_type_name bus_type_table[] = + { + {UNKNOWN_BUSTYPE, "CBUS "}, + {UNKNOWN_BUSTYPE, "CBUSII"}, + {EISA, "EISA "}, + {UNKNOWN_BUSTYPE, "FUTURE"}, + {UNKNOWN_BUSTYPE, "INTERN"}, + {ISA, "ISA "}, + {UNKNOWN_BUSTYPE, "MBI "}, + {UNKNOWN_BUSTYPE, "MBII "}, + {MCA, "MCA "}, + {UNKNOWN_BUSTYPE, "MPI "}, + {UNKNOWN_BUSTYPE, "MPSA "}, + {UNKNOWN_BUSTYPE, "NUBUS "}, + {PCI, "PCI "}, + {UNKNOWN_BUSTYPE, "PCMCIA"}, + {UNKNOWN_BUSTYPE, "TC "}, + {UNKNOWN_BUSTYPE, "VL "}, + {UNKNOWN_BUSTYPE, "VME "}, + {UNKNOWN_BUSTYPE, "XPRESS"} + }; + + /* From MP spec v1.4, table 5-1. */ + static int default_data[7][5] = + { + /* nbus, id0, type0, id1, type1 */ + {1, 0, ISA, 255, NOBUS}, + {1, 0, EISA, 255, NOBUS}, + {1, 0, EISA, 255, NOBUS}, + {1, 0, MCA, 255, NOBUS}, + {2, 0, ISA, 1, PCI}, + {2, 0, EISA, 1, PCI}, + {2, 0, MCA, 1, PCI} + }; + + struct pci_probe_table_args { + u_char bus; + u_char found; + }; + + struct pci_route_interrupt_args { + u_char bus; /* Source bus. */ + u_char irq; /* Source slot:pin. */ + int vector; /* Return value. */ + }; + + static mpfps_t mpfps; + static mpcth_t mpct; + static void *ioapics[NAPICID]; + static bus_datum *busses; + static int mptable_nioapics, mptable_nbusses, mptable_maxbusid; + static int pci0 = -1; + + MALLOC_DEFINE(M_MPTABLE, "MP Table", "MP Table Items"); + + static u_char conforming_polarity(u_char src_bus); + static u_char conforming_trigger(u_char src_bus, u_char src_bus_irq); + static u_char intentry_polarity(int_entry_ptr intr); + static u_char intentry_trigger(int_entry_ptr intr); + static int lookup_bus_type(char *name); + static void mptable_count_items(void); + static void mptable_count_items_handler(u_char *entry, void *arg); + static void mptable_hyperthread_fixup(u_int id_mask); + static void mptable_parse_apics_and_busses(void); + static void mptable_parse_apics_and_busses_handler(u_char *entry, + void *arg); + static void mptable_parse_ints(void); + static void mptable_parse_ints_handler(u_char *entry, void *arg); + static void mptable_parse_io_int(int_entry_ptr intr); + static void mptable_parse_local_int(int_entry_ptr intr); + static void mptable_pci_probe_table_handler(u_char *entry, void *arg); + static void mptable_pci_route_interrupt_handler(u_char *entry, void *arg); + static void mptable_pci_setup(void); + static int mptable_probe(void); + static int mptable_probe_cpus(void); + static void mptable_probe_cpus_handler(u_char *entry, void *arg __unused); + static void mptable_register(void *dummy); + static int mptable_setup_local(void); + static int mptable_setup_io(void); + static void mptable_walk_table(mptable_entry_handler *handler, void *arg); + static int search_for_sig(u_int32_t target, int count); + + static struct apic_enumerator mptable_enumerator = { + "MPTable", + mptable_probe, + mptable_probe_cpus, + mptable_setup_local, + mptable_setup_io + }; + + /* + * look for the MP spec signature + */ + + static int + search_for_sig(u_int32_t target, int count) + { + int x; + u_int32_t *addr = (u_int32_t *) (KERNBASE + target); + + for (x = 0; x < count; x += 4) + if (addr[x] == MP_SIG) + /* make array index a byte index */ + return (target + (x * sizeof(u_int32_t))); + return (-1); + } + + static int + lookup_bus_type(char *name) + { + int x; + + for (x = 0; x < MAX_BUSTYPE; ++x) + if (strncmp(bus_type_table[x].name, name, 6) == 0) + return (bus_type_table[x].type); + + return (UNKNOWN_BUSTYPE); + } + + /* + * Look for an Intel MP spec table (ie, SMP capable hardware). + */ + static int + mptable_probe(void) + { + int x; + u_long segment; + u_int32_t target; + + #if 0 + POSTCODE(MP_PROBE_POST); + #endif + + /* see if EBDA exists */ + if ((segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) != 0) { + /* search first 1K of EBDA */ + target = (u_int32_t) (segment << 4); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } else { + /* last 1K of base memory, effective 'top of base' passed in */ + target = (u_int32_t) (basemem - 0x400); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } + + /* search the BIOS */ + target = (u_int32_t) BIOS_BASE; + if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) + goto found; + + /* nothing found */ + return (ENXIO); + + found: + mpfps = (mpfps_t)(KERNBASE + x); + + /* Map in the configuration table if it exists. */ + if (mpfps->config_type != 0) + mpct = NULL; + else { + if ((uintptr_t)mpfps->pap >= 1024 * 1024) { + printf("%s: Unable to map MP Configuration Table\n", + __func__); + return (ENXIO); + } + mpct = (mpcth_t)(KERNBASE + (uintptr_t)mpfps->pap); + if (mpct->base_table_length + (uintptr_t)mpfps->pap >= + 1024 * 1024) { + printf("%s: Unable to map end of MP Config Table\n", + __func__); + return (ENXIO); + } + if (mpct->signature[0] != 'P' || mpct->signature[1] != 'C' || + mpct->signature[2] != 'M' || mpct->signature[3] != 'P') { + printf("%s: MP Config Table has bad signature: %c%c%c%c\n", + __func__, mpct->signature[0], mpct->signature[1], + mpct->signature[2], mpct->signature[3]); + return (ENXIO); + } + #if 0 + if (bootverbose) + #endif + printf( + "MP Configuration Table version 1.%d found at %p\n", + mpct->spec_rev, mpct); + } + + return (-100); + } + + /* + * Run through the MP table enumerating CPUs. + */ + static int + mptable_probe_cpus(void) + { + u_int cpu_mask; + + /* Is this a pre-defined config? */ + if (mpfps->config_type != 0) { + lapic_create(0, 1); + lapic_create(1, 0); + } else { + cpu_mask = 0; + mptable_walk_table(mptable_probe_cpus_handler, &cpu_mask); + mptable_hyperthread_fixup(cpu_mask); + } + return (0); + } + + /* + * Initialize the local APIC on the BSP. + */ + static int + mptable_setup_local(void) + { + + /* Is this a pre-defined config? */ + printf("MPTable: <"); + if (mpfps->config_type != 0) { + lapic_init(DEFAULT_APIC_BASE); + printf("Preset Config %d", mpfps->config_type); + } else { + lapic_init((uintptr_t)mpct->apic_address); + printf("%.*s %.*s", sizeof(mpct->oem_id), mpct->oem_id, + sizeof(mpct->product_id), mpct->product_id); + } + printf(">\n"); + return (0); + } + + /* + * Run through the MP table enumerating I/O APICs. + */ + static int + mptable_setup_io(void) + { + int i; + u_char byte; + + /* First, we count individual items and allocate arrays. */ + mptable_count_items(); + busses = malloc((mptable_maxbusid + 1) * sizeof(bus_datum), M_MPTABLE, + M_WAITOK); + for (i = 0; i <= mptable_maxbusid; i++) + busses[i].bus_type = NOBUS; + + /* Second, we run through adding I/O APIC's and busses. */ + mptable_parse_apics_and_busses(); + + /* Third, we run through the table tweaking interrupt sources. */ + mptable_parse_ints(); + + /* Fourth, we register all the I/O APIC's. */ + for (i = 0; i < NAPICID; i++) + if (ioapics[i] != NULL) + ioapic_register(ioapics[i]); + + /* Fifth, we setup data structures to handle PCI interrupt routing. */ + mptable_pci_setup(); + + /* Finally, we throw the switch to enable the I/O APIC's. */ + if (mpfps->mpfb2 & MPFB2_IMCR_PRESENT) { + outb(0x22, 0x70); /* select IMCR */ + byte = inb(0x23); /* current contents */ + byte |= 0x01; /* mask external INTR */ + outb(0x23, byte); /* disconnect 8259s/NMI */ + } + + return (0); + } + + static void + mptable_register(void *dummy __unused) + { + + apic_register_enumerator(&mptable_enumerator); + } + SYSINIT(mptable_register, SI_SUB_TUNABLES - 1, SI_ORDER_FIRST, + mptable_register, NULL) + + /* + * Call the handler routine for each entry in the MP config table. + */ + static void + mptable_walk_table(mptable_entry_handler *handler, void *arg) + { + u_int i; + u_char *entry; + + entry = (u_char *)(mpct + 1); + for (i = 0; i < mpct->entry_count; i++) { + switch (*entry) { + case MPCT_ENTRY_PROCESSOR: + case MPCT_ENTRY_IOAPIC: + case MPCT_ENTRY_BUS: + case MPCT_ENTRY_INT: + case MPCT_ENTRY_LOCAL_INT: + break; + default: + panic("%s: Unknown MP Config Entry %d\n", __func__, + (int)*entry); + } + handler(entry, arg); + entry += basetable_entry_types[*entry].length; + } + } + + static void + mptable_probe_cpus_handler(u_char *entry, void *arg) + { + proc_entry_ptr proc; + u_int *cpu_mask; + + switch (*entry) { + case MPCT_ENTRY_PROCESSOR: + proc = (proc_entry_ptr)entry; + if (proc->cpu_flags & PROCENTRY_FLAG_EN) { + lapic_create(proc->apic_id, proc->cpu_flags & + PROCENTRY_FLAG_BP); + cpu_mask = (u_int *)arg; + *cpu_mask |= (1 << proc->apic_id); + } + break; + } + } + + static void + mptable_count_items_handler(u_char *entry, void *arg __unused) + { + io_apic_entry_ptr apic; + bus_entry_ptr bus; + + switch (*entry) { + case MPCT_ENTRY_BUS: + bus = (bus_entry_ptr)entry; + mptable_nbusses++; + if (bus->bus_id > mptable_maxbusid) + mptable_maxbusid = bus->bus_id; + break; + case MPCT_ENTRY_IOAPIC: + apic = (io_apic_entry_ptr)entry; + if (apic->apic_flags & IOAPICENTRY_FLAG_EN) + mptable_nioapics++; + break; + } + } + + /* + * Count items in the table. + */ + static void + mptable_count_items(void) + { + + /* Is this a pre-defined config? */ + if (mpfps->config_type != 0) { + mptable_nioapics = 1; + switch (mpfps->config_type) { + case 1: + case 2: + case 3: + case 4: + mptable_nbusses = 1; + break; + case 5: + case 6: + case 7: + mptable_nbusses = 2; + break; + default: + panic("Unknown pre-defined MP Table config type %d", + mpfps->config_type); + } + mptable_maxbusid = mptable_nbusses - 1; + } else + mptable_walk_table(mptable_count_items_handler, NULL); + } + + /* + * Add a bus or I/O APIC from an entry in the table. + */ + static void + mptable_parse_apics_and_busses_handler(u_char *entry, void *arg __unused) + { + io_apic_entry_ptr apic; + bus_entry_ptr bus; + enum busTypes bus_type; + int i; + + + switch (*entry) { + case MPCT_ENTRY_BUS: + bus = (bus_entry_ptr)entry; + bus_type = lookup_bus_type(bus->bus_type); + if (bus_type == UNKNOWN_BUSTYPE) { + printf("MPTable: Unknown bus %d type \"", bus->bus_id); + for (i = 0; i < 6; i++) + printf("%c", bus->bus_type[i]); + printf("\"\n"); + } + busses[bus->bus_id].bus_id = bus->bus_id; + busses[bus->bus_id].bus_type = bus_type; + break; + case MPCT_ENTRY_IOAPIC: + apic = (io_apic_entry_ptr)entry; + if (!(apic->apic_flags & IOAPICENTRY_FLAG_EN)) + break; + if (apic->apic_id >= NAPICID) + panic("%s: I/O APIC ID %d too high", __func__, + apic->apic_id); + if (ioapics[apic->apic_id] != NULL) + panic("%s: Double APIC ID %d", __func__, + apic->apic_id); + ioapics[apic->apic_id] = ioapic_create( + (uintptr_t)apic->apic_address, apic->apic_id, -1); + break; + default: + break; + } + } + + /* + * Enumerate I/O APIC's and busses. + */ + static void + mptable_parse_apics_and_busses(void) + { + + /* Is this a pre-defined config? */ + if (mpfps->config_type != 0) { + ioapics[0] = ioapic_create(DEFAULT_IO_APIC_BASE, 2, 0); + busses[0].bus_id = 0; + busses[0].bus_type = default_data[mpfps->config_type][2]; + if (mptable_nbusses > 1) { + busses[1].bus_id = 1; + busses[1].bus_type = + default_data[mpfps->config_type][4]; + } + } else + mptable_walk_table(mptable_parse_apics_and_busses_handler, + NULL); + } + + /* + * Determine conforming polarity for a given bus type. + */ + static u_char + conforming_polarity(u_char src_bus) + { + + KASSERT(src_bus <= mptable_maxbusid, ("bus id %d too large", src_bus)); + switch (busses[src_bus].bus_type) { + case ISA: + case EISA: + /* Active Hi */ + return (1); + case PCI: + /* Active Lo */ + return (0); + default: + panic("%s: unknown bus type %d", __func__, + busses[src_bus].bus_type); + } + } + + /* + * Determine conforming trigger for a given bus type. + */ + static u_char + conforming_trigger(u_char src_bus, u_char src_bus_irq) + { + static int eisa_int_control = -1; + + KASSERT(src_bus <= mptable_maxbusid, ("bus id %d too large", src_bus)); + switch (busses[src_bus].bus_type) { + case ISA: + /* Edge Triggered */ + return (1); + case PCI: + /* Level Triggered */ + return (0); + case EISA: + KASSERT(src_bus_irq < 16, ("Invalid EISA IRQ %d", src_bus_irq)); + if (eisa_int_control == -1) + eisa_int_control = inb(ELCR1) << 8 | inb(ELCR0); + if (eisa_int_control & (1 << src_bus_irq)) + /* Level Triggered */ + return (0); + else + /* Edge Triggered */ + return (1); + default: + panic("%s: unknown bus type %d", __func__, + busses[src_bus].bus_type); + } + } + + static u_char + intentry_polarity(int_entry_ptr intr) + { + + switch (intr->int_flags & INTENTRY_FLAGS_POLARITY) { + case INTENTRY_FLAGS_POLARITY_CONFORM: + return (conforming_polarity(intr->src_bus_id)); + case INTENTRY_FLAGS_POLARITY_ACTIVEHI: + return (1); + case INTENTRY_FLAGS_POLARITY_ACTIVELO: + return (0); + default: + panic("Bogus interrupt flags"); + } + } + + static u_char + intentry_trigger(int_entry_ptr intr) + { + + switch (intr->int_flags & INTENTRY_FLAGS_TRIGGER) { + case INTENTRY_FLAGS_TRIGGER_CONFORM: + return (conforming_trigger(intr->src_bus_id, + intr->src_bus_irq)); + case INTENTRY_FLAGS_TRIGGER_EDGE: + return (1); + case INTENTRY_FLAGS_TRIGGER_LEVEL: + return (0); + default: + panic("Bogus interrupt flags"); + } + } + + /* + * Parse an interrupt entry for an I/O interrupt routed to a pin on an I/O APIC. + */ + static void + mptable_parse_io_int(int_entry_ptr intr) + { + void *ioapic; + u_int pin; + + if (intr->dst_apic_id == 0xff) { + printf("MPTable: Ignoring global interrupt entry for pin %d\n", + intr->dst_apic_int); + return; + } + if (intr->dst_apic_id >= NAPICID) { + printf("MPTable: Ignoring interrupt entry for ioapic%d\n", + intr->dst_apic_id); + return; + } + ioapic = ioapics[intr->dst_apic_id]; + if (ioapic == NULL) { + printf( + "MPTable: Ignoring interrupt entry for missing ioapic%d\n", + intr->dst_apic_id); + return; + } + pin = intr->dst_apic_int; + switch (intr->int_type) { + case INTENTRY_TYPE_INT: + if (busses[intr->src_bus_id].bus_type == NOBUS) + panic("interrupt from missing bus"); + if (busses[intr->src_bus_id].bus_type == ISA && + intr->src_bus_irq != pin) + ioapic_remap_vector(ioapic, pin, intr->src_bus_irq); + break; + case INTENTRY_TYPE_NMI: + ioapic_set_nmi(ioapic, pin); + break; + case INTENTRY_TYPE_SMI: + ioapic_set_smi(ioapic, pin); + break; + case INTENTRY_TYPE_EXTINT: + ioapic_set_extint(ioapic, pin); + break; + default: + panic("%s: invalid interrupt entry type %d\n", __func__, + intr->int_type); + } + if (intr->int_type == INTENTRY_TYPE_INT || + (intr->int_flags & INTENTRY_FLAGS_TRIGGER) != + INTENTRY_FLAGS_TRIGGER_CONFORM) + ioapic_set_triggermode(ioapic, pin, intentry_trigger(intr)); + if (intr->int_type == INTENTRY_TYPE_INT || + (intr->int_flags & INTENTRY_FLAGS_POLARITY) != + INTENTRY_FLAGS_POLARITY_CONFORM) + ioapic_set_polarity(ioapic, pin, intentry_polarity(intr)); + } + + /* + * Parse an interrupt entry for a local APIC LVT pin. + */ + static void + mptable_parse_local_int(int_entry_ptr intr) + { + u_int apic_id, pin; + + if (intr->dst_apic_id == 0xff) + apic_id = APIC_ID_ALL; + else + apic_id = intr->dst_apic_id; + if (intr->dst_apic_int == 0) + pin = LVT_LINT0; + else + pin = LVT_LINT1; + switch (intr->int_type) { + case INTENTRY_TYPE_INT: + #if 1 + printf( + "MPTable: Ignoring vectored local interrupt for LINTIN%d vector %d\n", + intr->dst_apic_int, intr->src_bus_irq); + return; + #else + lapic_set_lvt_mode(apic_id, pin, APIC_LVT_DM_FIXED); + break; + #endif + case INTENTRY_TYPE_NMI: + lapic_set_lvt_mode(apic_id, pin, APIC_LVT_DM_NMI); + break; + case INTENTRY_TYPE_SMI: + lapic_set_lvt_mode(apic_id, pin, APIC_LVT_DM_SMI); + break; + case INTENTRY_TYPE_EXTINT: + lapic_set_lvt_mode(apic_id, pin, APIC_LVT_DM_EXTINT); + break; + default: + panic("%s: invalid interrupt entry type %d\n", __func__, + intr->int_type); + } + if ((intr->int_flags & INTENTRY_FLAGS_TRIGGER) != + INTENTRY_FLAGS_TRIGGER_CONFORM) + lapic_set_lvt_triggermode(apic_id, pin, + intentry_trigger(intr)); + if ((intr->int_flags & INTENTRY_FLAGS_POLARITY) != + INTENTRY_FLAGS_POLARITY_CONFORM) + lapic_set_lvt_polarity(apic_id, pin, intentry_polarity(intr)); + } + + /* + * Parse interrupt entries. + */ + static void + mptable_parse_ints_handler(u_char *entry, void *arg __unused) + { + int_entry_ptr intr; + + intr = (int_entry_ptr)entry; + switch (*entry) { + case MPCT_ENTRY_INT: + mptable_parse_io_int(intr); + break; + case MPCT_ENTRY_LOCAL_INT: + mptable_parse_local_int(intr); + break; + } + } + + /* + * Configure the interrupt pins + */ + static void + mptable_parse_ints(void) + { + + /* Is this a pre-defined config? */ + if (mpfps->config_type != 0) { + /* Configure LINT pins. */ + lapic_set_lvt_mode(APIC_ID_ALL, LVT_LINT0, APIC_LVT_DM_EXTINT); + lapic_set_lvt_mode(APIC_ID_ALL, LVT_LINT1, APIC_LVT_DM_NMI); + + /* Configure I/O APIC pins. */ + if (mpfps->config_type != 7) + ioapic_set_extint(ioapics[0], 0); + else + ioapic_disable_pin(ioapics[0], 0); + if (mpfps->config_type != 2) + ioapic_remap_vector(ioapics[0], 2, 0); + else + ioapic_disable_pin(ioapics[0], 2); + if (mpfps->config_type == 2) + ioapic_disable_pin(ioapics[0], 13); + } else + mptable_walk_table(mptable_parse_ints_handler, NULL); + } + + /* + * Perform a hyperthreading "fix-up" to enumerate any logical CPU's + * that aren't already listed in the table. + * + * XXX: We assume that all of the physical CPUs in the + * system have the same number of logical CPUs. + * + * XXX: We assume that APIC ID's are allocated such that + * the APIC ID's for a physical processor are aligned + * with the number of logical CPU's in the processor. + */ + static void + mptable_hyperthread_fixup(u_int id_mask) + { + u_int i, id, logical_cpus; + + /* Nothing to do if there is no HTT support. */ + if ((cpu_feature & CPUID_HTT) == 0) + return; + logical_cpus = (cpu_procinfo & CPUID_HTT_CORES) >> 16; + if (logical_cpus <= 1) + return; + + /* + * For each APIC ID of a CPU that is set in the mask, + * scan the other candidate APIC ID's for this + * physical processor. If any of those ID's are + * already in the table, then kill the fixup. + */ + for (id = 0; id <= MAXCPU; id++) { + if ((id_mask & 1 << id) == 0) + continue; + /* First, make sure we are on a logical_cpus boundary. */ + if (id % logical_cpus != 0) + return; + for (i = id + 1; i < id + logical_cpus; i++) + if ((id_mask & 1 << i) != 0) + return; + } + + /* + * Ok, the ID's checked out, so perform the fixup by + * adding the logical CPUs. + */ + while ((id = ffs(id_mask)) != 0) { + id--; + for (i = id + 1; i < id + logical_cpus; i++) { + #if 0 + if (bootverbose) + #endif + printf( + "MPTable: Adding logical CPU %d from main CPU %d\n", + i, id); + lapic_create(i, 0); + } + id_mask &= ~(1 << id); + } + } + + /* + * Support code for routing PCI interrupts using the MP Table. + */ + static void + mptable_pci_setup(void) + { + int i; + + /* + * Find the first pci bus and call it 0. Panic if pci0 is not + * bus zero and there are multiple PCI busses. + */ + for (i = 0; i <= mptable_maxbusid; i++) + if (busses[i].bus_type == PCI) { + if (pci0 == -1) + pci0 = i; + else if (pci0 != 0) + panic( + "MPTable contains multiple PCI busses but no PCI bus 0"); + } + } + + static void + mptable_pci_probe_table_handler(u_char *entry, void *arg) + { + struct pci_probe_table_args *args; + int_entry_ptr intr; + + if (*entry != MPCT_ENTRY_INT) + return; + intr = (int_entry_ptr)entry; + args = (struct pci_probe_table_args *)arg; + KASSERT(args->bus <= mptable_maxbusid, + ("bus %d is too big", args->bus)); + KASSERT(busses[args->bus].bus_type == PCI, ("probing for non-PCI bus")); + if (intr->src_bus_id == args->bus) + args->found = 1; + } + + int + mptable_pci_probe_table(int bus) + { + struct pci_probe_table_args args; + + if (bus < 0) + return (EINVAL); + if (pci0 == -1 || pci0 + bus > mptable_maxbusid) + return (ENXIO); + args.bus = pci0 + bus; + args.found = 0; + mptable_walk_table(mptable_pci_probe_table_handler, &args); + if (args.found == 0) + return (ENXIO); + return (0); + } + + static void + mptable_pci_route_interrupt_handler(u_char *entry, void *arg) + { + struct pci_route_interrupt_args *args; + int_entry_ptr intr; + + if (*entry != MPCT_ENTRY_INT) + return; + intr = (int_entry_ptr)entry; + args = (struct pci_route_interrupt_args *)arg; + if (intr->src_bus_id != args->bus || intr->src_bus_irq != args->irq) + return; + KASSERT(args->vector == -1, + ("Multiple entries for PCI IRQ %d", args->vector)); + KASSERT(ioapics[intr->dst_apic_id] != NULL, + ("No I/O APIC %d to route interrupt to", intr->dst_apic_id)); + args->vector = ioapic_get_vector(ioapics[intr->dst_apic_id], + intr->dst_apic_int); + } + + int + mptable_pci_route_interrupt(device_t pcib, device_t dev, int pin) + { + struct pci_route_interrupt_args args; + int slot; + + /* Like ACPI, pin numbers are 0-3, not 1-4. */ + pin--; + KASSERT(pci0 != -1, ("do not know how to route PCI interrupts")); + args.bus = pci_get_bus(dev) + pci0; + slot = pci_get_slot(dev); + + /* + * PCI interrupt entries in the MP Table encode both the slot and + * pin into the IRQ with the pin being the two least significant + * bits, the slot being the next five bits, and the most significant + * bit being reserved. + */ + args.irq = slot << 2 | pin; + args.vector = -1; + mptable_walk_table(mptable_pci_route_interrupt_handler, &args); + if (args.vector < 0) { + device_printf(pcib, "unable to route slot %d INT%c\n", slot, + 'A' + pin); + return (PCI_INVALID_IRQ); + } + device_printf(pcib, "slot %d INT%c routed to irq %d\n", slot, 'A' + pin, + args.vector); + return (args.vector); + } Index: i386/i386/mptable_pci.c =========================================================================== *** /dev/null Thu Oct 30 14:47:44 2003 --- i386/i386/mptable_pci.c Thu Oct 30 14:48:09 2003 *************** *** 0 **** --- 1,159 ---- + /*- + * Copyright (c) 2003 John Baldwin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the author nor the names of any co-contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + /* + * Host to PCI and PCI to PCI bridge drivers that use the MP Table to route + * interrupts from PCI devices to I/O APICs. + */ + + #include + __FBSDID("$FreeBSD$"); + + #include + #include + #include + #include + #include + + #include + #include + #include + #include + #include + #include + + #include "pcib_if.h" + + /* Host to PCI bridge driver. */ + + static int + mptable_hostb_probe(device_t dev) + { + + if (pci_cfgregopen() == 0) + return (ENXIO); + if (mptable_pci_probe_table(pcib_get_bus(dev)) != 0) + return (ENXIO); + device_set_desc(dev, "MPTable Host-PCI bridge"); + return (0); + } + + static device_method_t mptable_hostb_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, mptable_hostb_probe), + DEVMETHOD(device_attach, legacy_pcib_attach), + DEVMETHOD(device_shutdown, bus_generic_shutdown), + DEVMETHOD(device_suspend, bus_generic_suspend), + DEVMETHOD(device_resume, bus_generic_resume), + + /* Bus interface */ + DEVMETHOD(bus_print_child, bus_generic_print_child), + DEVMETHOD(bus_read_ivar, legacy_pcib_read_ivar), + DEVMETHOD(bus_write_ivar, legacy_pcib_write_ivar), + DEVMETHOD(bus_alloc_resource, bus_generic_alloc_resource), + DEVMETHOD(bus_release_resource, bus_generic_release_resource), + DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), + DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), + DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), + DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), + + /* pcib interface */ + DEVMETHOD(pcib_maxslots, legacy_pcib_maxslots), + DEVMETHOD(pcib_read_config, legacy_pcib_read_config), + DEVMETHOD(pcib_write_config, legacy_pcib_write_config), + DEVMETHOD(pcib_route_interrupt, mptable_pci_route_interrupt), + + { 0, 0 } + }; + + static driver_t mptable_hostb_driver = { + "pcib", + mptable_hostb_methods, + 1, + }; + + DRIVER_MODULE(mptable_pcib, legacy, mptable_hostb_driver, pcib_devclass, 0, 0); + + /* PCI to PCI bridge driver. */ + + static int + mptable_pcib_probe(device_t dev) + { + int bus; + + if ((pci_get_class(dev) != PCIC_BRIDGE) || + (pci_get_subclass(dev) != PCIS_BRIDGE_PCI)) + return (ENXIO); + bus = pci_read_config(dev, PCIR_SECBUS_1, 1); + if (bus == 0) + return (ENXIO); + if (mptable_pci_probe_table(bus) != 0) + return (ENXIO); + device_set_desc(dev, "MPTable PCI-PCI bridge"); + return (-1000); + } + + static device_method_t mptable_pcib_pci_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, mptable_pcib_probe), + DEVMETHOD(device_attach, pcib_attach), + DEVMETHOD(device_shutdown, bus_generic_shutdown), + DEVMETHOD(device_suspend, bus_generic_suspend), + DEVMETHOD(device_resume, bus_generic_resume), + + /* Bus interface */ + DEVMETHOD(bus_print_child, bus_generic_print_child), + DEVMETHOD(bus_read_ivar, pcib_read_ivar), + DEVMETHOD(bus_write_ivar, pcib_write_ivar), + DEVMETHOD(bus_alloc_resource, pcib_alloc_resource), + DEVMETHOD(bus_release_resource, bus_generic_release_resource), + DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), + DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), + DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), + DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), + + /* pcib interface */ + DEVMETHOD(pcib_maxslots, pcib_maxslots), + DEVMETHOD(pcib_read_config, pcib_read_config), + DEVMETHOD(pcib_write_config, pcib_write_config), + DEVMETHOD(pcib_route_interrupt, mptable_pci_route_interrupt), + + {0, 0} + }; + + static driver_t mptable_pcib_driver = { + "pcib", + mptable_pcib_pci_methods, + sizeof(struct pcib_softc), + }; + + DRIVER_MODULE(mptable_pcib, pci, mptable_pcib_driver, pcib_devclass, 0, 0); + Index: i386/i386/nexus.c =========================================================================== --- sys/i386/i386/nexus.c 2003/10/30 14:48:07 #10 +++ i386/i386/nexus.c 2003/10/30 14:48:07 @@ -50,6 +50,7 @@ #include #include #include +#include #include #include @@ -59,10 +60,6 @@ #include #include -#ifdef APIC_IO -#include -#include -#endif #ifdef DEV_ISA #include @@ -72,8 +69,6 @@ #include #endif #endif -#include -#include #include static MALLOC_DEFINE(M_NEXUSDEV, "nexusdev", "Nexus device"); @@ -164,18 +159,11 @@ * multi-ISA-bus systems. PCI interrupts are routed to the ISA * component, so in a way, PCI can be a partial child of an ISA bus(!). * APIC interrupts are global though. - * - * XXX We depend on the AT PIC driver correctly claiming IRQ 2 - * to prevent its reuse elsewhere in the !APIC_IO case. */ irq_rman.rm_start = 0; irq_rman.rm_type = RMAN_ARRAY; irq_rman.rm_descr = "Interrupt request lines"; -#ifdef APIC_IO - irq_rman.rm_end = APIC_INTMAPSIZE - 1; -#else - irq_rman.rm_end = 15; -#endif + irq_rman.rm_end = NUM_IO_INTS - 1; if (rman_init(&irq_rman) || rman_manage_region(&irq_rman, irq_rman.rm_start, irq_rman.rm_end)) @@ -474,7 +462,7 @@ if (error) return (error); - error = inthand_add(device_get_nameunit(child), irq->r_start, + error = intr_add_handler(device_get_nameunit(child), irq->r_start, ihand, arg, flags, cookiep); return (error); @@ -483,7 +471,7 @@ static int nexus_teardown_intr(device_t dev, device_t child, struct resource *r, void *ih) { - return (inthand_remove(ih)); + return (intr_remove_handler(ih)); } static int Index: i386/i386/pmap.c =========================================================================== --- sys/i386/i386/pmap.c 2003/10/30 14:48:07 #165 +++ i386/i386/pmap.c 2003/10/30 14:48:07 @@ -136,12 +136,9 @@ #include #include #include -#if defined(SMP) || defined(APIC_IO) +#ifdef SMP #include -#include -#include -#include -#endif /* SMP || APIC_IO */ +#endif #define PMAP_KEEP_PDIRS #ifndef PMAP_SHPGPERPROC @@ -391,16 +388,6 @@ /* Turn on PG_G on kernel page(s) */ pmap_set_pg(); - -#ifdef SMP - if (cpu_apic_address == 0) - panic("pmap_bootstrap: no local apic! (non-SMP hardware?)"); - - /* local apic is mapped on last page */ - SMPpt[NPTEPG - 1] = (pt_entry_t)(PG_V | PG_RW | PG_N | pgeflag | - (cpu_apic_address & PG_FRAME)); -#endif - invltlb(); } /* @@ -603,11 +590,17 @@ u_int cpumask; u_int other_cpus; - critical_enter(); + if (smp_started) { + if (!(read_eflags() & PSL_I)) + panic("%s: interrupts disabled", __func__); + mtx_lock_spin(&smp_tlb_mtx); + } else + critical_enter(); /* * We need to disable interrupt preemption but MUST NOT have * interrupts disabled here. * XXX we may need to hold schedlock to get a coherent pm_active + * XXX critical sections disable interrupts again */ if (pmap->pm_active == -1 || pmap->pm_active == all_cpus) { invlpg(va); @@ -620,7 +613,10 @@ if (pmap->pm_active & other_cpus) smp_masked_invlpg(pmap->pm_active & other_cpus, va); } - critical_exit(); + if (smp_started) + mtx_unlock_spin(&smp_tlb_mtx); + else + critical_exit(); } void @@ -630,11 +626,17 @@ u_int other_cpus; vm_offset_t addr; - critical_enter(); + if (smp_started) { + if (!(read_eflags() & PSL_I)) + panic("%s: interrupts disabled", __func__); + mtx_lock_spin(&smp_tlb_mtx); + } else + critical_enter(); /* * We need to disable interrupt preemption but MUST NOT have * interrupts disabled here. * XXX we may need to hold schedlock to get a coherent pm_active + * XXX critical sections disable interrupts again */ if (pmap->pm_active == -1 || pmap->pm_active == all_cpus) { for (addr = sva; addr < eva; addr += PAGE_SIZE) @@ -650,7 +652,10 @@ smp_masked_invlpg_range(pmap->pm_active & other_cpus, sva, eva); } - critical_exit(); + if (smp_started) + mtx_unlock_spin(&smp_tlb_mtx); + else + critical_exit(); } void @@ -659,11 +664,17 @@ u_int cpumask; u_int other_cpus; - critical_enter(); + if (smp_started) { + if (!(read_eflags() & PSL_I)) + panic("%s: interrupts disabled", __func__); + mtx_lock_spin(&smp_tlb_mtx); + } else + critical_enter(); /* * We need to disable interrupt preemption but MUST NOT have * interrupts disabled here. * XXX we may need to hold schedlock to get a coherent pm_active + * XXX critical sections disable interrupts again */ if (pmap->pm_active == -1 || pmap->pm_active == all_cpus) { invltlb(); @@ -676,7 +687,10 @@ if (pmap->pm_active & other_cpus) smp_masked_invltlb(pmap->pm_active & other_cpus); } - critical_exit(); + if (smp_started) + mtx_unlock_spin(&smp_tlb_mtx); + else + critical_exit(); } #else /* !SMP */ /* @@ -2882,7 +2896,10 @@ offset = pa & PAGE_MASK; size = roundup(offset + size, PAGE_SIZE); - va = kmem_alloc_nofault(kernel_map, size); + if (pa < KERNLOAD && pa + size <= KERNLOAD) + va = KERNBASE + pa; + else + va = kmem_alloc_nofault(kernel_map, size); if (!va) panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); @@ -2904,6 +2921,8 @@ { vm_offset_t base, offset, tmpva; + if (va >= KERNBASE && va + size <= KERNBASE + KERNLOAD) + return; base = va & PG_FRAME; offset = va & PAGE_MASK; size = roundup(offset + size, PAGE_SIZE); Index: i386/i386/support.s =========================================================================== --- sys/i386/i386/support.s 2003/10/30 14:48:07 #16 +++ i386/i386/support.s 2003/10/30 14:48:07 @@ -37,6 +37,7 @@ #include #include +#include #include #include @@ -62,6 +63,16 @@ .byte 0xfe .space 3 #endif + ALIGN_DATA + .globl intrcnt, eintrcnt +intrcnt: + .space INTRCNT_COUNT * 4 +eintrcnt: + + .globl intrnames, eintrnames +intrnames: + .space INTRCNT_COUNT * (MAXCOMLEN + 1) +eintrnames: .text Index: i386/i386/trap.c =========================================================================== --- sys/i386/i386/trap.c 2003/10/30 14:48:07 #62 +++ i386/i386/trap.c 2003/10/30 14:48:07 @@ -82,6 +82,7 @@ #include #include +#include #include #include #ifdef SMP @@ -90,9 +91,6 @@ #include #include -#include -#include - #ifdef POWERFAIL_NMI #include #include @@ -764,7 +762,7 @@ #ifdef SMP /* two separate prints in case of a trap on an unmapped page */ printf("cpuid = %d; ", PCPU_GET(cpuid)); - printf("lapic.id = %08x\n", lapic.id); + printf("apic id = %02x\n", PCPU_GET(apic_id)); #endif if (type == T_PAGEFLT) { printf("fault virtual address = 0x%x\n", eva); @@ -847,7 +845,7 @@ #ifdef SMP /* two separate prints in case of a trap on an unmapped page */ printf("cpuid = %d; ", PCPU_GET(cpuid)); - printf("lapic.id = %08x\n", lapic.id); + printf("apic id = %02x\n", PCPU_GET(apic_id)); #endif panic("double fault"); } Index: i386/include/apic.h =========================================================================== --- sys/i386/include/apic.h 2003/10/30 14:48:07 #8 +++ i386/include/apic.h 2003/10/30 14:48:07 @@ -25,8 +25,8 @@ * $FreeBSD: src/sys/i386/include/apic.h,v 1.24 2003/09/10 01:11:58 jhb Exp $ */ -#ifndef _MACHINE_APIC_H_ -#define _MACHINE_APIC_H_ +#ifndef _MACHINE_APICREG_H_ +#define _MACHINE_APICREG_H_ /* * Local && I/O APIC definitions. @@ -221,11 +221,29 @@ /* default physical locations of LOCAL (CPU) APICs */ #define DEFAULT_APIC_BASE 0xfee00000 +/* constants relating to APIC ID registers */ +#define APIC_ID_MASK 0xff000000 +#define APIC_ID_SHIFT 24 +#define APIC_ID_CLUSTER 0xf0 +#define APIC_ID_CLUSTER_ID 0x0f +#define APIC_MAX_CLUSTER 0xe +#define APIC_MAX_INTRACLUSTER_ID 3 +#define APIC_ID_CLUSTER_SHIFT 4 + /* fields in VER */ #define APIC_VER_VERSION 0x000000ff #define APIC_VER_MAXLVT 0x00ff0000 #define MAXLVTSHIFT 16 +/* fields in LDR */ +#define APIC_LDR_RESERVED 0x00ffffff + +/* fields in DFR */ +#define APIC_DFR_RESERVED 0x0fffffff +#define APIC_DFR_MODEL_MASK 0xf0000000 +#define APIC_DFR_MODEL_FLAT 0xf0000000 +#define APIC_DFR_MODEL_CLUSTER 0x00000000 + /* fields in SVR */ #define APIC_SVR_VECTOR 0x000000ff #define APIC_SVR_VEC_PROG 0x000000f0 @@ -290,10 +308,6 @@ #define APIC_ICRLO_RESV_MASK (APIC_RESV1_MASK | APIC_RESV2_MASK) -/* fields in ICR_HIGH */ -#define APIC_ID_MASK 0xff000000 -#define APIC_ID_SHIFT 24 - /* fields in LVT1/2 */ #define APIC_LVT_VECTOR 0x000000ff #define APIC_LVT_DM 0x00000700 @@ -444,4 +458,4 @@ #define IOART_INTVEC 0x000000ff /* R/W: INTerrupt vector field */ -#endif /* _MACHINE_APIC_H_ */ +#endif /* _MACHINE_APICREG_H_ */ Index: i386/include/apicvar.h =========================================================================== *** /dev/null Thu Oct 30 14:47:44 2003 --- i386/include/apicvar.h Thu Oct 30 14:48:10 2003 *************** *** 0 **** --- 1,165 ---- + /*- + * Copyright (c) 2003 John Baldwin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the author nor the names of any co-contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + #ifndef _MACHINE_APICVAR_H_ + #define _MACHINE_APICVAR_H_ + + /* + * Local && I/O APIC variable definitions. + */ + + /* + * Layout of local APIC interrupt vectors: + * + * 0xff (255) +-------------+ + * | | 15 (Spurios Vector) + * 0xf0 (240) +-------------+ + * | | 14 (Interprocessor Interrupts) + * 0xe0 (224) +-------------+ + * | | 13 (Local Interrupt (LINT[01])) + * 0xd0 (208) +-------------+ + * | | 12 (Local Timer and Error Interrupts) + * 0xc0 (192) +-------------+ + * | | 11 (I/O Interrupts) + * 0xb0 (176) +-------------+ + * | | 10 (I/O Interrupts) + * 0xa0 (160) +-------------+ + * | | 9 (I/O Interrupts) + * 0x90 (144) +-------------+ + * | | 8 (I/O Interrupts / System Calls) + * 0x80 (128) +-------------+ + * | | 7 (I/O Interrupts) + * 0x70 (112) +-------------+ + * | | 6 (I/O Interrupts) + * 0x60 (96) +-------------+ + * | | 5 (I/O Interrupts) + * 0x50 (80) +-------------+ + * | | 4 (I/O Interrupts) + * 0x40 (64) +-------------+ + * | | 3 (I/O Interrupts) + * 0x30 (48) +-------------+ + * | | 2 (I/O Interrupts) + * 0x20 (32) +-------------+ + * | | 1 (Exceptions, traps, faults, etc.) + * 0x10 (16) +-------------+ + * | | 0 (Exceptions, traps, faults, etc.) + * 0x00 (0) +-------------+ + * + * Note: 0x80 needs to be handled specially and not allocated to an + * I/O device! + */ + + #define APIC_ID_ALL 0xff + #define APIC_NUM_IOINTS 160 + + #define APIC_LOCAL_INTS (IDT_IO_INTS + APIC_NUM_IOINTS) + #define APIC_TIMER_INT APIC_LOCAL_INTS + #define APIC_ERROR_INT (APIC_LOCAL_INTS + 1) + #define APIC_THERMAL_INT (APIC_LOCAL_INTS + 2) + + #define APIC_IPI_INTS (APIC_LOCAL_INTS + 32) + #define IPI_AST APIC_IPI_INTS /* Generate software trap. */ + #define IPI_INVLTLB (APIC_IPI_INTS + 1) /* TLB Shootdown IPIs */ + #define IPI_INVLPG (APIC_IPI_INTS + 2) + #define IPI_INVLRNG (APIC_IPI_INTS + 3) + #define IPI_HARDCLOCK (APIC_IPI_INTS + 8) /* Inter-CPU clock handling. */ + #define IPI_STATCLOCK (APIC_IPI_INTS + 9) + #define IPI_RENDEZVOUS (APIC_IPI_INTS + 10) /* Inter-CPU rendezvous. */ + #define IPI_LAZYPMAP (APIC_IPI_INTS + 11) /* Lazy pmap release. */ + #define IPI_STOP (APIC_IPI_INTS + 12) /* Stop CPU until restarted. */ + + #define APIC_SPURIOUS_INT 255 + + #define LVT_LINT0 0 + #define LVT_LINT1 1 + #define LVT_TIMER 2 + #define LVT_ERROR 3 + #define LVT_PMC 4 + #define LVT_THERMAL 5 + #define LVT_MAX LVT_THERMAL + + #ifndef LOCORE + + #define APIC_IPI_DEST_SELF -1 + #define APIC_IPI_DEST_ALL -2 + #define APIC_IPI_DEST_OTHERS -3 + + /* + * An APIC enumerator is a psuedo bus driver that enumerates APIC's including + * CPU's and I/O APIC's. + */ + struct apic_enumerator { + const char *apic_name; + int (*apic_probe)(void); + int (*apic_probe_cpus)(void); + int (*apic_setup_local)(void); + int (*apic_setup_io)(void); + SLIST_ENTRY(apic_enumerator) apic_next; + }; + + inthand_t + IDTVEC(apic_isr1), IDTVEC(apic_isr2), IDTVEC(apic_isr3), + IDTVEC(apic_isr4), IDTVEC(apic_isr5), IDTVEC(spuriousint); + + u_int apic_irq_to_idt(u_int irq); + u_int apic_idt_to_irq(u_int vector); + void apic_register_enumerator(struct apic_enumerator *enumerator); + void *ioapic_create(uintptr_t addr, int32_t id, int intbase); + int ioapic_disable_pin(void *cookie, u_int pin); + int ioapic_get_vector(void *cookie, u_int pin); + int ioapic_next_logical_cluster(void); + void ioapic_register(void *cookie); + int ioapic_remap_vector(void *cookie, u_int pin, int vector); + int ioapic_set_extint(void *cookie, u_int pin); + int ioapic_set_nmi(void *cookie, u_int pin); + int ioapic_set_polarity(void *cookie, u_int pin, char activehi); + int ioapic_set_triggermode(void *cookie, u_int pin, char edgetrigger); + int ioapic_set_smi(void *cookie, u_int pin); + void lapic_create(u_int apic_id, int boot_cpu); + void lapic_disable(void); + void lapic_dump(const char *str); + void lapic_enable_intr(u_int vector); + int lapic_id(void); + void lapic_init(uintptr_t addr); + int lapic_intr_pending(u_int vector); + void lapic_ipi_raw(register_t icrlo, u_int dest); + void lapic_ipi_vectored(u_int vector, int dest); + int lapic_ipi_wait(int delay); + void lapic_handle_intr(struct intrframe frame); + void lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id); + int lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked); + int lapic_set_lvt_mode(u_int apic_id, u_int lvt, u_int32_t mode); + int lapic_set_lvt_polarity(u_int apic_id, u_int lvt, u_char activehi); + int lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, u_char edgetrigger); + void lapic_setup(void); + + #endif /* !LOCORE */ + #endif /* _MACHINE_APICVAR_H_ */ Index: i386/include/asmacros.h =========================================================================== --- sys/i386/include/asmacros.h 2003/10/30 14:48:07 #3 +++ i386/include/asmacros.h 2003/10/30 14:48:07 @@ -138,4 +138,15 @@ #define MEXITCOUNT #endif /* GPROF */ +#ifdef LOCORE +/* + * Convenience macros for declaring interrupt entry points and trap + * stubs. + */ +#define IDTVEC(name) ALIGN_TEXT; .globl __CONCAT(X,name); \ + .type __CONCAT(X,name),@function; __CONCAT(X,name): +#define TRAP(a) pushl $(a) ; jmp alltraps + +#endif /* LOCORE */ + #endif /* !_MACHINE_ASMACROS_H_ */ Index: i386/include/clock.h =========================================================================== --- sys/i386/include/clock.h 2003/10/30 14:48:07 #9 +++ i386/include/clock.h 2003/10/30 14:48:07 @@ -24,9 +24,6 @@ extern uint64_t tsc_freq; extern int tsc_is_broken; extern int wall_cmos_clock; -#ifdef APIC_IO -extern int apic_8254_intr; -#endif /* * Driver to clock driver interface. Index: i386/include/critical.h =========================================================================== --- sys/i386/include/critical.h 2003/10/30 14:48:07 #4 +++ i386/include/critical.h 2003/10/30 14:48:07 @@ -44,7 +44,6 @@ /* * Prototypes - see //critical.c */ -void cpu_unpend(void); void cpu_critical_fork_exit(void); #ifdef __GNUC__ @@ -59,7 +58,11 @@ * However, as a side effect any interrupts occuring while td_critnest * is non-zero will be deferred. */ -#define cpu_critical_enter() +static __inline void +cpu_critical_enter(void) +{ + curthread->td_md.md_savecrit = intr_disable(); +} /* * cpu_critical_exit: @@ -75,19 +78,7 @@ static __inline void cpu_critical_exit(void) { - /* - * We may have to schedule pending interrupts. Create - * conditions similar to an interrupt context and call - * unpend(). - * - * note: we do this even if we are in an interrupt - * nesting level. Deep nesting is protected by - * critical_*() and if we conditionalized it then we - * would have to check int_pending again whenever - * we decrement td_intr_nesting_level to 0. - */ - if (PCPU_GET(int_pending)) - cpu_unpend(); + intr_restore(curthread->td_md.md_savecrit); } #else /* !__GNUC__ */ Index: i386/include/intr_machdep.h =========================================================================== *** /dev/null Thu Oct 30 14:47:44 2003 --- i386/include/intr_machdep.h Thu Oct 30 14:48:10 2003 *************** *** 0 **** --- 1,91 ---- + /*- + * Copyright (c) 2003 John Baldwin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + #ifndef __MACHINE_INTR_MACHDEP_H__ + #define __MACHINE_INTR_MACHDEP_H__ + + #ifdef _KERNEL + + /* With I/O APIC's we can have up to 159 interrupts. */ + #define NUM_IO_INTS 159 + #define INTRCNT_COUNT (1 + NUM_IO_INTS * 2) + + #ifndef LOCORE + + typedef void inthand_t(u_int cs, u_int ef, u_int esp, u_int ss); + + #define IDTVEC(name) __CONCAT(X,name) + + struct intsrc; + + /* + * Methods that a PIC provides to mask/unmask a given interrupt source, + * "turn on" the interrupt on the CPU side by setting up an IDT entry, and + * return the vector associated with this source. + */ + struct pic { + void (*pic_enable_source)(struct intsrc *); + void (*pic_disable_source)(struct intsrc *); + void (*pic_eoi_source)(struct intsrc *); + void (*pic_enable_intr)(struct intsrc *); + int (*pic_vector)(struct intsrc *); + int (*pic_source_pending)(struct intsrc *); + void (*pic_suspend)(struct intsrc *); + void (*pic_resume)(struct intsrc *); + }; + + /* + * An interrupt source. The upper-layer code uses the PIC methods to + * control a given source. The lower-layer PIC drivers can store additional + * private data in a given interrupt source such as an interrupt pin number + * or an I/O APIC pointer. + */ + struct intsrc { + struct pic *is_pic; + struct ithd *is_ithread; + u_long *is_count; + u_long *is_straycount; + u_int is_index; + }; + + struct intrframe; + + extern struct mtx icu_lock; + + int intr_add_handler(const char *name, int vector, driver_intr_t handler, + void *arg, enum intr_type flags, void **cookiep); + void intr_execute_handlers(struct intsrc *isrc, struct intrframe *iframe); + struct intsrc *intr_lookup_source(int vector); + int intr_register_source(struct intsrc *isrc); + int intr_remove_handler(void *cookie); + void intr_resume(void); + void intr_suspend(void); + + #endif /* !LOCORE */ + #endif /* _KERNEL */ + #endif /* !__MACHINE_INTR_MACHDEP_H__ */ Index: i386/include/md_var.h =========================================================================== --- sys/i386/include/md_var.h 2003/10/30 14:48:07 #26 +++ i386/include/md_var.h 2003/10/30 14:48:07 @@ -43,6 +43,7 @@ extern long Maxmem; extern u_int atdevbase; /* offset in virtual memory of ISA io mem */ +extern u_int basemem; /* PA of original top of base memory */ extern int busdma_swi_pending; extern u_int cpu_exthigh; extern u_int cpu_feature; @@ -96,6 +97,7 @@ void sse2_pagezero(void *addr); void init_AMD_Elan_sc520(void); int is_physical_memory(vm_offset_t addr); +int isa_nmi(int cd); vm_paddr_t kvtop(void *addr); void setidt(int idx, alias_for_inthand_t *func, int typ, int dpl, int selec); int user_dbreg_trap(void); Index: i386/include/mpapic.h =========================================================================== --- sys/i386/include/mpapic.h 2003/10/30 14:48:07 #1 +++ i386/include/mpapic.h 2003/10/30 14:48:07 @@ -28,6 +28,8 @@ #ifndef _MACHINE_MPAPIC_H_ #define _MACHINE_MPAPIC_H_ +#error Don't use this header + /* * Size of APIC ID list. * Also used a MAX size of various other arrays. Index: i386/include/mptable.h =========================================================================== *** /dev/null Thu Oct 30 14:47:44 2003 --- i386/include/mptable.h Thu Oct 30 14:48:10 2003 *************** *** 0 **** --- 1,145 ---- + /* + * Copyright (c) 1996, by Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + #ifndef __MACHINE_MPTABLE_H__ + #define __MACHINE_MPTABLE_H__ + + enum busTypes { + NOBUS = 0, + EISA = 3, + ISA = 6, + MCA = 9, + PCI = 13, + MAX_BUSTYPE = 18, + UNKNOWN_BUSTYPE = 0xff + }; + + /* MP Floating Pointer Structure */ + typedef struct MPFPS { + char signature[4]; + void *pap; + u_char length; + u_char spec_rev; + u_char checksum; + u_char config_type; + u_char mpfb2; + u_char mpfb3; + u_char mpfb4; + u_char mpfb5; + } *mpfps_t; + + #define MPFB2_IMCR_PRESENT 0x80 + #define MPFB2_MUL_CLK_SRCS 0x40 + + /* MP Configuration Table Header */ + typedef struct MPCTH { + char signature[4]; + u_short base_table_length; + u_char spec_rev; + u_char checksum; + u_char oem_id[8]; + u_char product_id[12]; + void *oem_table_pointer; + u_short oem_table_size; + u_short entry_count; + void *apic_address; + u_short extended_table_length; + u_char extended_table_checksum; + u_char reserved; + } *mpcth_t; + + #define MPCT_ENTRY_PROCESSOR 0 + #define MPCT_ENTRY_BUS 1 + #define MPCT_ENTRY_IOAPIC 2 + #define MPCT_ENTRY_INT 3 + #define MPCT_ENTRY_LOCAL_INT 4 + + typedef struct PROCENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char cpu_flags; + u_long cpu_signature; + u_long feature_flags; + u_long reserved1; + u_long reserved2; + } *proc_entry_ptr; + + #define PROCENTRY_FLAG_EN 0x01 + #define PROCENTRY_FLAG_BP 0x02 + + typedef struct BUSENTRY { + u_char type; + u_char bus_id; + char bus_type[6]; + } *bus_entry_ptr; + + typedef struct IOAPICENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char apic_flags; + void *apic_address; + } *io_apic_entry_ptr; + + #define IOAPICENTRY_FLAG_EN 0x01 + + typedef struct INTENTRY { + u_char type; + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; + } *int_entry_ptr; + + #define INTENTRY_TYPE_INT 0 + #define INTENTRY_TYPE_NMI 1 + #define INTENTRY_TYPE_SMI 2 + #define INTENTRY_TYPE_EXTINT 3 + + #define INTENTRY_FLAGS_POLARITY 0x3 + #define INTENTRY_FLAGS_POLARITY_CONFORM 0x0 + #define INTENTRY_FLAGS_POLARITY_ACTIVEHI 0x1 + #define INTENTRY_FLAGS_POLARITY_ACTIVELO 0x3 + #define INTENTRY_FLAGS_TRIGGER 0xc + #define INTENTRY_FLAGS_TRIGGER_CONFORM 0x0 + #define INTENTRY_FLAGS_TRIGGER_EDGE 0x4 + #define INTENTRY_FLAGS_TRIGGER_LEVEL 0xc + + /* descriptions of MP basetable entries */ + typedef struct BASETABLE_ENTRY { + u_char type; + u_char length; + char name[16]; + } basetable_entry; + + int mptable_pci_probe_table(int bus); + int mptable_pci_route_interrupt(device_t pcib, device_t dev, int pin); + + #endif /* !__MACHINE_MPTABLE_H__ */ Index: i386/include/pcpu.h =========================================================================== --- sys/i386/include/pcpu.h 2003/10/30 14:48:07 #7 +++ i386/include/pcpu.h 2003/10/30 14:48:07 @@ -48,10 +48,7 @@ struct segment_descriptor pc_common_tssd; \ struct segment_descriptor *pc_tss_gdt; \ int pc_currentldt; \ - u_int32_t pc_int_pending; /* master int pending flag */ \ - u_int32_t pc_ipending; /* pending slow interrupts */ \ - u_int32_t pc_fpending; /* pending fast interrupts */ \ - u_int32_t pc_spending /* pending soft interrupts */ + u_int pc_apic_id; #if defined(lint) Index: i386/include/proc.h =========================================================================== --- sys/i386/include/proc.h 2003/10/30 14:48:07 #9 +++ i386/include/proc.h 2003/10/30 14:48:07 @@ -51,9 +51,7 @@ * Machine-dependent part of the proc structure for i386. */ struct mdthread { -#ifdef lint - int dummy; -#endif + register_t md_savecrit; }; struct mdproc { Index: i386/include/segments.h =========================================================================== --- sys/i386/include/segments.h 2003/10/30 14:48:07 #10 +++ i386/include/segments.h 2003/10/30 14:48:07 @@ -173,11 +173,7 @@ * Size of IDT table */ -#if defined(SMP) || defined(APIC_IO) -#define NIDT 256 /* we use them all */ -#else -#define NIDT 129 /* 32 reserved, 16 h/w, 0 s/w, linux's 0x80 */ -#endif /* SMP || APIC_IO */ +#define NIDT 256 /* 32 reserved, 0x80 syscall, most are h/w */ #define NRSVIDT 32 /* reserved entries for cpu exceptions */ /* @@ -202,6 +198,7 @@ #define IDT_AC 17 /* #AC: Alignment Check */ #define IDT_MC 18 /* #MC: Machine Check */ #define IDT_XF 19 /* #XF: SIMD Floating-Point Exception */ +#define IDT_IO_INTS NRSVIDT /* Base of IDT entries for I/O interrupts. */ #define IDT_SYSCALL 0x80 /* System Call Interrupt Vector */ /* Index: i386/include/smp.h =========================================================================== --- sys/i386/include/smp.h 2003/10/30 14:48:07 #12 +++ i386/include/smp.h 2003/10/30 14:48:07 @@ -15,17 +15,7 @@ #ifdef _KERNEL -#if defined(SMP) && defined(I386_CPU) && !defined(COMPILING_LINT) -#error SMP not supported with I386_CPU -#endif -#if defined(SMP) && !defined(APIC_IO) -# error APIC_IO required for SMP, add "options APIC_IO" to your config file. -#endif /* SMP && !APIC_IO */ -#if defined(SMP) && defined(CPU_DISABLE_CMPXCHG) && !defined(COMPILING_LINT) -#error SMP not supported with CPU_DISABLE_CMPXCHG -#endif - -#if defined(SMP) || defined(APIC_IO) +#ifdef SMP #ifndef LOCORE @@ -43,26 +33,11 @@ current_postcode |= (((X) << 4) & 0xf0), \ outb(0x80, current_postcode) - -#include /* XXX */ -#include +#include #include -#include -#include +#include +#include -/* - * Interprocessor interrupts for SMP. - */ -#define IPI_INVLTLB XINVLTLB_OFFSET -#define IPI_INVLPG XINVLPG_OFFSET -#define IPI_INVLRNG XINVLRNG_OFFSET -#define IPI_LAZYPMAP XLAZYPMAP_OFFSET -#define IPI_RENDEZVOUS XRENDEZVOUS_OFFSET -#define IPI_AST XCPUAST_OFFSET -#define IPI_STOP XCPUSTOP_OFFSET -#define IPI_HARDCLOCK XHARDCLOCK_OFFSET -#define IPI_STATCLOCK XSTATCLOCK_OFFSET - /* global data in mpboot.s */ extern int bootMP_size; @@ -70,61 +45,36 @@ void bootMP(void); /* global data in mp_machdep.c */ -extern int bsp_apic_ready; extern int mp_naps; -extern int mp_nbusses; -extern int mp_napics; -extern int mp_picmode; extern int boot_cpu_id; -extern vm_offset_t cpu_apic_address; -extern vm_offset_t io_apic_address[]; -extern u_int32_t cpu_apic_versions[]; -extern u_int32_t *io_apic_versions; -extern int cpu_num_to_apic_id[]; -extern int io_num_to_apic_id[]; -extern int apic_id_to_logical[]; -#define APIC_INTMAPSIZE 32 -struct apic_intmapinfo { - int ioapic; - int int_pin; - volatile void *apic_address; - int redirindex; -}; -extern struct apic_intmapinfo int_to_apicintpin[]; extern struct pcb stoppcbs[]; +extern struct mtx smp_tlb_mtx; + +/* IPI handlers */ +inthand_t + IDTVEC(invltlb), /* TLB shootdowns - global */ + IDTVEC(invlpg), /* TLB shootdowns - 1 page */ + IDTVEC(invlrng), /* TLB shootdowns - page range */ + IDTVEC(hardclock), /* Forward hardclock() */ + IDTVEC(statclock), /* Forward statclock() */ + IDTVEC(cpuast), /* Additional software trap on other cpu */ + IDTVEC(cpustop), /* CPU stops & waits to be restarted */ + IDTVEC(rendezvous), /* handle CPU rendezvous */ + IDTVEC(lazypmap); /* handle lazy pmap release */ /* functions in mp_machdep.c */ -void i386_mp_probe(void); -u_int mp_bootaddress(u_int); -u_int isa_apic_mask(u_int); -int isa_apic_irq(int); -int pci_apic_irq(int, int, int); -int apic_irq(int, int); -int next_apic_irq(int); -int undirect_isa_irq(int); -int undirect_pci_irq(int); -int apic_bus_type(int); -int apic_src_bus_id(int, int); -int apic_src_bus_irq(int, int); -int apic_int_type(int, int); -int apic_trigger(int, int); -int apic_polarity(int, int); -int mp_grab_cpu_hlt(void); -void assign_apic_irq(int apic, int intpin, int irq); -void revoke_apic_irq(int irq); -void bsp_apic_configure(void); +void cpu_add(u_int apic_id, char boot_cpu); void init_secondary(void); +void ipi_selected(u_int cpus, u_int ipi); +void ipi_all(u_int ipi); +void ipi_all_but_self(u_int ipi); +void ipi_self(u_int ipi); void forward_statclock(void); void forwarded_statclock(struct clockframe frame); void forward_hardclock(void); void forwarded_hardclock(struct clockframe frame); -void ipi_selected(u_int cpus, u_int ipi); -void ipi_all(u_int ipi); -void ipi_all_but_self(u_int ipi); -void ipi_self(u_int ipi); -#ifdef APIC_INTR_REORDER -void set_lapic_isrloc(int, int); -#endif /* APIC_INTR_REORDER */ +u_int mp_bootaddress(u_int); +int mp_grab_cpu_hlt(void); void smp_invlpg(vm_offset_t addr); void smp_masked_invlpg(u_int mask, vm_offset_t addr); void smp_invlpg_range(vm_offset_t startva, vm_offset_t endva); @@ -133,30 +83,8 @@ void smp_invltlb(void); void smp_masked_invltlb(u_int mask); -/* global data in mpapic.c */ -extern volatile lapic_t lapic; -extern volatile ioapic_t **ioapic; - -/* functions in mpapic.c */ -void apic_dump(char*); -void apic_initialize(void); -void imen_dump(void); -int apic_ipi(int, int, int); -int selected_apic_ipi(u_int, int, int); -int io_apic_setup(int); -void io_apic_setup_intpin(int, int); -void io_apic_set_id(int, int); -int io_apic_get_id(int); -int ext_int_setup(int, int); - -void set_apic_timer(int); -int read_apic_timer(void); -void u_sleep(int); -u_int io_apic_read(int, int); -void io_apic_write(int, int, u_int); - #endif /* !LOCORE */ -#endif /* SMP && !APIC_IO */ +#endif /* SMP */ #endif /* _KERNEL */ #endif /* _MACHINE_SMP_H_ */ Index: i386/include/smptests.h =========================================================================== --- sys/i386/include/smptests.h 2003/10/30 14:48:07 #5 +++ i386/include/smptests.h 2003/10/30 14:48:07 @@ -33,15 +33,7 @@ * Various 'tests in progress' and configuration parameters. */ - /* - * Put FAST_INTR() ISRs at an APIC priority above the regular INTs. - * Allow the mp_lock() routines to handle FAST interrupts while spinning. - */ -#define FAST_HI - - -/* * These defines enable critical region locking of areas that were * protected via cli/sti in the UP kernel. * @@ -59,39 +51,12 @@ */ #define CPUSTOP_ON_DDBBREAK - -#ifdef APIC_IO -/* - * Don't assume that slow interrupt handler X is called from vector - * X + ICU_OFFSET. - */ -#define APIC_INTR_REORDER - -#endif /* APIC_IO */ - /* * Misc. counters. * #define COUNT_XINVLTLB_HITS */ - -/** - * Hack to "fake-out" kernel into thinking it is running on a 'default config'. - * - * value == default type -#define TEST_DEFAULT_CONFIG 6 - */ - - -/* - * Simple test code for IPI interaction, save for future... - * -#define TEST_TEST1 -#define IPI_TARGET_TEST1 1 - */ - - /* * Address of POST hardware port. * Defining this enables POSTCODE macros. Index: i386/isa/apic_vector.s =========================================================================== --- sys/i386/isa/apic_vector.s 2003/10/30 14:48:07 #20 +++ i386/isa/apic_vector.s 2003/10/30 14:48:07 @@ -1,19 +1,52 @@ -/* +/*- + * Copyright (c) 1989, 1990 William F. Jolitz. + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * * from: vector.s, 386BSD 0.1 unknown origin * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.90 2003/10/16 10:44:24 bde Exp $ */ -#include -#include +/* + * Interrupt entry points for external interrupts triggered by I/O APICs + * as well as IPI handlers. + */ -/* convert an absolute IRQ# into a bitmask */ -#define IRQ_BIT(irq_num) (1 << (irq_num)) +#include +#include -/* make an index into the IO APIC from the IRQ# */ -#define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2)) +#include "assym.s" /* - * + * Macros to create and destroy a trap frame. */ #define PUSH_FRAME \ pushl $0 ; /* dummy error code */ \ @@ -23,14 +56,6 @@ pushl %es ; \ pushl %fs -#define PUSH_DUMMY \ - pushfl ; /* eflags */ \ - pushl %cs ; /* cs */ \ - pushl 12(%esp) ; /* original caller eip */ \ - pushl $0 ; /* dummy error code */ \ - pushl $0 ; /* dummy trap type */ \ - subl $11*4,%esp ; - #define POP_FRAME \ popl %fs ; \ popl %es ; \ @@ -38,209 +63,40 @@ popal ; \ addl $4+4,%esp -#define POP_DUMMY \ - addl $16*4,%esp - -#define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8 -#define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12 - -#define MASK_IRQ(irq_num) \ - ICU_LOCK ; /* into critical reg */ \ - testl $IRQ_BIT(irq_num), apic_imen ; \ - jne 7f ; /* masked, don't mask */ \ - orl $IRQ_BIT(irq_num), apic_imen ; /* set the mask bit */ \ - movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \ - movl REDIRIDX(irq_num), %eax ; /* get the index */ \ - movl %eax, (%ecx) ; /* write the index */ \ - movl IOAPIC_WINDOW(%ecx), %eax ; /* current value */ \ - orl $IOART_INTMASK, %eax ; /* set the mask */ \ - movl %eax, IOAPIC_WINDOW(%ecx) ; /* new value */ \ -7: ; /* already masked */ \ - ICU_UNLOCK /* - * Test to see whether we are handling an edge or level triggered INT. - * Level-triggered INTs must still be masked as we don't clear the source, - * and the EOI cycle would cause redundant INTs to occur. + * I/O Interrupt Entry Point. Rather than having one entry point for + * each interrupt source, we use one entry point for each 32-bit word + * in the ISR. The handler determines the highest bit set in the ISR, + * translates that into a vector, and passes the vector to the + * lapic_handle_intr() function. */ -#define MASK_LEVEL_IRQ(irq_num) \ - testl $IRQ_BIT(irq_num), apic_pin_trigger ; \ - jz 9f ; /* edge, don't mask */ \ - MASK_IRQ(irq_num) ; \ -9: - - -#ifdef APIC_INTR_REORDER -#define EOI_IRQ(irq_num) \ - movl apic_isrbit_location + 8 * (irq_num), %eax ; \ - movl (%eax), %eax ; \ - testl apic_isrbit_location + 4 + 8 * (irq_num), %eax ; \ - jz 9f ; /* not active */ \ - movl $0, lapic+LA_EOI ; \ -9: - -#else -#define EOI_IRQ(irq_num) \ - testl $IRQ_BIT(irq_num), lapic+LA_ISR1; \ - jz 9f ; /* not active */ \ - movl $0, lapic+LA_EOI; \ -9: -#endif - - -/* - * Test to see if the source is currently masked, clear if so. - */ -#define UNMASK_IRQ(irq_num) \ - ICU_LOCK ; /* into critical reg */ \ - testl $IRQ_BIT(irq_num), apic_imen ; \ - je 7f ; /* bit clear, not masked */ \ - andl $~IRQ_BIT(irq_num), apic_imen ;/* clear mask bit */ \ - movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \ - movl REDIRIDX(irq_num), %eax ; /* get the index */ \ - movl %eax, (%ecx) ; /* write the index */ \ - movl IOAPIC_WINDOW(%ecx), %eax ; /* current value */ \ - andl $~IOART_INTMASK, %eax ; /* clear the mask */ \ - movl %eax, IOAPIC_WINDOW(%ecx) ; /* new value */ \ -7: ; /* already unmasked */ \ - ICU_UNLOCK - -/* - * Test to see whether we are handling an edge or level triggered INT. - * Level-triggered INTs have to be unmasked. - */ -#define UNMASK_LEVEL_IRQ(irq_num) \ - testl $IRQ_BIT(irq_num), apic_pin_trigger ; \ - jz 9f ; /* edge, don't unmask */ \ - UNMASK_IRQ(irq_num) ; \ -9: - -/* - * Macros for interrupt entry, call to handler, and exit. - */ - -#define FAST_INTR(irq_num, vec_name) \ - .text ; \ - SUPERALIGN_TEXT ; \ -IDTVEC(vec_name) ; \ - PUSH_FRAME ; \ - movl $KDSEL,%eax ; \ - mov %ax,%ds ; \ - mov %ax,%es ; \ - movl $KPSEL,%eax ; \ - mov %ax,%fs ; \ - FAKE_MCOUNT(13*4(%esp)) ; \ - movl PCPU(CURTHREAD),%ebx ; \ - cmpl $0,TD_CRITNEST(%ebx) ; \ - je 1f ; \ -; \ - movl $1,PCPU(INT_PENDING) ; \ - orl $IRQ_BIT(irq_num),PCPU(FPENDING) ; \ - MASK_LEVEL_IRQ(irq_num) ; \ - movl $0, lapic+LA_EOI ; \ - jmp 10f ; \ -1: ; \ - incl TD_CRITNEST(%ebx) ; \ - incl TD_INTR_NESTING_LEVEL(%ebx) ; \ - pushl intr_unit + (irq_num) * 4 ; \ - call *intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ - addl $4, %esp ; \ - movl $0, lapic+LA_EOI ; \ - lock ; \ - incl cnt+V_INTR ; /* book-keeping can wait */ \ - movl intr_countp + (irq_num) * 4, %eax ; \ - lock ; \ - incl (%eax) ; \ - decl TD_CRITNEST(%ebx) ; \ - cmpl $0,PCPU(INT_PENDING) ; \ - je 2f ; \ -; \ - call i386_unpend ; \ -2: ; \ - decl TD_INTR_NESTING_LEVEL(%ebx) ; \ -10: ; \ - MEXITCOUNT ; \ - jmp doreti - -/* - * Restart a fast interrupt that was held up by a critical section. - * This routine is called from unpend(). unpend() ensures we are - * in a critical section and deals with the interrupt nesting level - * for us. If we previously masked the irq, we have to unmask it. - * - * We have a choice. We can regenerate the irq using the 'int' - * instruction or we can create a dummy frame and call the interrupt - * handler directly. I've chosen to use the dummy-frame method. - */ -#define FAST_UNPEND(irq_num, vec_name) \ - .text ; \ - SUPERALIGN_TEXT ; \ -IDTVEC(vec_name) ; \ -; \ - pushl %ebp ; \ - movl %esp, %ebp ; \ - PUSH_DUMMY ; \ - pushl intr_unit + (irq_num) * 4 ; \ - call *intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ - addl $4, %esp ; \ - lock ; \ - incl cnt+V_INTR ; /* book-keeping can wait */ \ - movl intr_countp + (irq_num) * 4, %eax ; \ - lock ; \ - incl (%eax) ; \ - UNMASK_LEVEL_IRQ(irq_num) ; \ - POP_DUMMY ; \ - popl %ebp ; \ - ret ; \ - - -/* - * Slow, threaded interrupts. - * - * XXX Most of the parameters here are obsolete. Fix this when we're - * done. - * XXX we really shouldn't return via doreti if we just schedule the - * interrupt handler and don't run anything. We could just do an - * iret. FIXME. - */ -#define INTR(irq_num, vec_name, maybe_extra_ipending) \ +#define ISR_VEC(index, vec_name) \ .text ; \ SUPERALIGN_TEXT ; \ -/* _XintrNN: entry point used by IDT/HWIs via _vec[]. */ \ IDTVEC(vec_name) ; \ PUSH_FRAME ; \ movl $KDSEL, %eax ; /* reload with kernel's data segment */ \ mov %ax, %ds ; \ mov %ax, %es ; \ - movl $KPSEL, %eax ; \ + movl $KPSEL, %eax ; /* reload with per-CPU data segment */ \ mov %ax, %fs ; \ -; \ - maybe_extra_ipending ; \ -; \ - MASK_LEVEL_IRQ(irq_num) ; \ - EOI_IRQ(irq_num) ; \ -; \ - movl PCPU(CURTHREAD),%ebx ; \ - cmpl $0,TD_CRITNEST(%ebx) ; \ - je 1f ; \ - movl $1,PCPU(INT_PENDING) ; \ - orl $IRQ_BIT(irq_num),PCPU(IPENDING) ; \ - jmp 10f ; \ + movl lapic, %edx ; /* pointer to local APIC */ \ + movl PCPU(CURTHREAD), %ebx ; \ + movl LA_ISR + 16 * (index)(%edx), %eax ; /* load ISR */ \ + incl TD_INTR_NESTING_LEVEL(%ebx) ; \ + bsrl %eax, %eax ; /* index of highset set bit in ISR */ \ + jz 2f ; \ + addl $(32 * index),%eax ; \ 1: ; \ - incl TD_INTR_NESTING_LEVEL(%ebx) ; \ -; \ - FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid dbl cnt */ \ - cmpl $0,PCPU(INT_PENDING) ; \ - je 9f ; \ - call i386_unpend ; \ -9: ; \ - pushl $irq_num; /* pass the IRQ */ \ - call sched_ithd ; \ - addl $4, %esp ; /* discard the parameter */ \ -; \ + FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid double count */ \ + pushl %eax ; /* pass the IRQ */ \ + call lapic_handle_intr ; \ + addl $4, %esp ; /* discard parameter */ \ decl TD_INTR_NESTING_LEVEL(%ebx) ; \ -10: ; \ MEXITCOUNT ; \ - jmp doreti + jmp doreti ; \ +2: movl $-1, %eax ; /* send a vector of -1 */ \ + jmp 1b /* * Handle "spurious INTerrupts". @@ -257,6 +113,14 @@ iret +MCOUNT_LABEL(bintr2) + ISR_VEC(1,apic_isr1) + ISR_VEC(2,apic_isr2) + ISR_VEC(3,apic_isr3) + ISR_VEC(4,apic_isr4) + ISR_VEC(5,apic_isr5) +MCOUNT_LABEL(eintr2) + #ifdef SMP /* * Global address space TLB shootdown. @@ -281,7 +145,8 @@ movl %cr3, %eax /* invalidate the TLB */ movl %eax, %cr3 - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %eax + movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ lock incl smp_tlb_wait @@ -313,7 +178,8 @@ movl smp_tlb_addr1, %eax invlpg (%eax) /* invalidate single page */ - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %eax + movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ lock incl smp_tlb_wait @@ -350,7 +216,8 @@ cmpl %eax, %edx jb 1b - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %eax + movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ lock incl smp_tlb_wait @@ -374,21 +241,15 @@ movl $KPSEL, %eax mov %ax, %fs - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %edx + movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC */ movl PCPU(CURTHREAD),%ebx - cmpl $0,TD_CRITNEST(%ebx) - je 1f - movl $1,PCPU(INT_PENDING) - orl $1,PCPU(SPENDING); - jmp 10f -1: incl TD_INTR_NESTING_LEVEL(%ebx) pushl $0 /* XXX convert trapframe to clockframe */ call forwarded_hardclock addl $4, %esp /* XXX convert clockframe to trapframe */ decl TD_INTR_NESTING_LEVEL(%ebx) -10: MEXITCOUNT jmp doreti @@ -406,23 +267,17 @@ movl $KPSEL, %eax mov %ax, %fs - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %edx + movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC */ FAKE_MCOUNT(13*4(%esp)) movl PCPU(CURTHREAD),%ebx - cmpl $0,TD_CRITNEST(%ebx) - je 1f - movl $1,PCPU(INT_PENDING) - orl $2,PCPU(SPENDING); - jmp 10f -1: incl TD_INTR_NESTING_LEVEL(%ebx) pushl $0 /* XXX convert trapframe to clockframe */ call forwarded_statclock addl $4, %esp /* XXX convert clockframe to trapframe */ decl TD_INTR_NESTING_LEVEL(%ebx) -10: MEXITCOUNT jmp doreti @@ -444,7 +299,8 @@ movl $KPSEL, %eax mov %ax, %fs - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %edx + movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC */ FAKE_MCOUNT(13*4(%esp)) @@ -476,7 +332,8 @@ movl $KPSEL, %eax mov %ax, %fs - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %eax + movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ movl PCPU(CPUID), %eax imull $PCB_SIZE, %eax @@ -518,111 +375,6 @@ popl %ebp iret -#endif /* SMP */ - -MCOUNT_LABEL(bintr) - FAST_INTR(0,fastintr0) - FAST_INTR(1,fastintr1) - FAST_INTR(2,fastintr2) - FAST_INTR(3,fastintr3) - FAST_INTR(4,fastintr4) - FAST_INTR(5,fastintr5) - FAST_INTR(6,fastintr6) - FAST_INTR(7,fastintr7) - FAST_INTR(8,fastintr8) - FAST_INTR(9,fastintr9) - FAST_INTR(10,fastintr10) - FAST_INTR(11,fastintr11) - FAST_INTR(12,fastintr12) - FAST_INTR(13,fastintr13) - FAST_INTR(14,fastintr14) - FAST_INTR(15,fastintr15) - FAST_INTR(16,fastintr16) - FAST_INTR(17,fastintr17) - FAST_INTR(18,fastintr18) - FAST_INTR(19,fastintr19) - FAST_INTR(20,fastintr20) - FAST_INTR(21,fastintr21) - FAST_INTR(22,fastintr22) - FAST_INTR(23,fastintr23) - FAST_INTR(24,fastintr24) - FAST_INTR(25,fastintr25) - FAST_INTR(26,fastintr26) - FAST_INTR(27,fastintr27) - FAST_INTR(28,fastintr28) - FAST_INTR(29,fastintr29) - FAST_INTR(30,fastintr30) - FAST_INTR(31,fastintr31) -#define CLKINTR_PENDING movl $1,CNAME(clkintr_pending) -/* Threaded interrupts */ - INTR(0,intr0, CLKINTR_PENDING) - INTR(1,intr1,) - INTR(2,intr2,) - INTR(3,intr3,) - INTR(4,intr4,) - INTR(5,intr5,) - INTR(6,intr6,) - INTR(7,intr7,) - INTR(8,intr8,) - INTR(9,intr9,) - INTR(10,intr10,) - INTR(11,intr11,) - INTR(12,intr12,) - INTR(13,intr13,) - INTR(14,intr14,) - INTR(15,intr15,) - INTR(16,intr16,) - INTR(17,intr17,) - INTR(18,intr18,) - INTR(19,intr19,) - INTR(20,intr20,) - INTR(21,intr21,) - INTR(22,intr22,) - INTR(23,intr23,) - INTR(24,intr24,) - INTR(25,intr25,) - INTR(26,intr26,) - INTR(27,intr27,) - INTR(28,intr28,) - INTR(29,intr29,) - INTR(30,intr30,) - INTR(31,intr31,) - - FAST_UNPEND(0,fastunpend0) - FAST_UNPEND(1,fastunpend1) - FAST_UNPEND(2,fastunpend2) - FAST_UNPEND(3,fastunpend3) - FAST_UNPEND(4,fastunpend4) - FAST_UNPEND(5,fastunpend5) - FAST_UNPEND(6,fastunpend6) - FAST_UNPEND(7,fastunpend7) - FAST_UNPEND(8,fastunpend8) - FAST_UNPEND(9,fastunpend9) - FAST_UNPEND(10,fastunpend10) - FAST_UNPEND(11,fastunpend11) - FAST_UNPEND(12,fastunpend12) - FAST_UNPEND(13,fastunpend13) - FAST_UNPEND(14,fastunpend14) - FAST_UNPEND(15,fastunpend15) - FAST_UNPEND(16,fastunpend16) - FAST_UNPEND(17,fastunpend17) - FAST_UNPEND(18,fastunpend18) - FAST_UNPEND(19,fastunpend19) - FAST_UNPEND(20,fastunpend20) - FAST_UNPEND(21,fastunpend21) - FAST_UNPEND(22,fastunpend22) - FAST_UNPEND(23,fastunpend23) - FAST_UNPEND(24,fastunpend24) - FAST_UNPEND(25,fastunpend25) - FAST_UNPEND(26,fastunpend26) - FAST_UNPEND(27,fastunpend27) - FAST_UNPEND(28,fastunpend28) - FAST_UNPEND(29,fastunpend29) - FAST_UNPEND(30,fastunpend30) - FAST_UNPEND(31,fastunpend31) -MCOUNT_LABEL(eintr) - -#ifdef SMP /* * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. * @@ -640,7 +392,8 @@ call smp_rendezvous_action - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %eax + movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ POP_FRAME iret @@ -658,16 +411,9 @@ mov %ax, %fs call pmap_lazyfix_action - - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + + movl lapic, %eax + movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ POP_FRAME iret #endif /* SMP */ - - .data - - .globl apic_pin_trigger -apic_pin_trigger: - .long 0 - - .text Index: i386/isa/atpic.c =========================================================================== *** /dev/null Thu Oct 30 14:47:44 2003 --- i386/isa/atpic.c Thu Oct 30 14:48:10 2003 *************** *** 0 **** --- 1,353 ---- + /*- + * Copyright (c) 2003 John Baldwin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the author nor the names of any co-contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + /* + * PIC driver for the 8259A Master and Slave PICs in PC/AT machines. + */ + + #include "opt_isa.h" + + #include + #include + #include + #include + #include + #include + #include + + #include + #include + #include + #include + #include + + #include + #include + #include + + #define MASTER 0 + #define SLAVE 1 + + /* XXX: Magic numbers */ + #ifdef PC98 + #ifdef AUTO_EOI_1 + #define MASTER_MODE 0x1f /* Master auto EOI, 8086 mode */ + #else + #define MASTER_MODE 0x1d /* Master 8086 mode */ + #endif + #define SLAVE_MODE 9 /* 8086 mode */ + #else /* IBM-PC */ + #ifdef AUTO_EOI_1 + #define MASTER_MODE (2 | 1) /* Auto EOI, 8086 mode */ + #else + #define MASTER_MODE 1 /* 8086 mode */ + #endif + #ifdef AUTO_EOI_2 + #define SLAVE_MODE (2 | 1) /* Auto EOI, 8086 mode */ + #else + #define SLAVE_MODE 1 /* 8086 mode */ + #endif + #endif /* PC98 */ + + static void atpic_init(void *dummy); + + unsigned int imen; /* XXX */ + + inthand_t + IDTVEC(atpic_intr0), IDTVEC(atpic_intr1), IDTVEC(atpic_intr2), + IDTVEC(atpic_intr3), IDTVEC(atpic_intr4), IDTVEC(atpic_intr5), + IDTVEC(atpic_intr6), IDTVEC(atpic_intr7), IDTVEC(atpic_intr8), + IDTVEC(atpic_intr9), IDTVEC(atpic_intr10), IDTVEC(atpic_intr11), + IDTVEC(atpic_intr12), IDTVEC(atpic_intr13), IDTVEC(atpic_intr14), + IDTVEC(atpic_intr15); + + #define IRQ(ap, ai) ((ap)->at_irqbase + (ai)->at_irq) + + #define ATPIC(io, base, eoi, imenptr) \ + { { atpic_enable_source, atpic_disable_source, (eoi), \ + atpic_enable_intr, atpic_vector, atpic_source_pending, NULL, \ + atpic_resume }, (io), (base), IDT_IO_INTS + (base), (imenptr) } + + #define INTSRC(irq) \ + { { &atpics[(irq) / 8].at_pic }, (irq) % 8, \ + IDTVEC(atpic_intr ## irq ) } + + struct atpic { + struct pic at_pic; + int at_ioaddr; + int at_irqbase; + uint8_t at_intbase; + uint8_t *at_imen; + }; + + struct atpic_intsrc { + struct intsrc at_intsrc; + int at_irq; /* Relative to PIC base. */ + inthand_t *at_intr; + }; + + static void atpic_enable_source(struct intsrc *isrc); + static void atpic_disable_source(struct intsrc *isrc); + static void atpic_eoi_master(struct intsrc *isrc); + static void atpic_eoi_slave(struct intsrc *isrc); + static void atpic_enable_intr(struct intsrc *isrc); + static int atpic_vector(struct intsrc *isrc); + static void atpic_resume(struct intsrc *isrc); + static int atpic_source_pending(struct intsrc *isrc); + static void i8259_init(struct atpic *pic, int slave); + + static struct atpic atpics[] = { + ATPIC(IO_ICU1, 0, atpic_eoi_master, (uint8_t *)&imen), + ATPIC(IO_ICU2, 8, atpic_eoi_slave, ((uint8_t *)&imen) + 1) + }; + + static struct atpic_intsrc atintrs[] = { + INTSRC(0), + INTSRC(1), + INTSRC(2), + INTSRC(3), + INTSRC(4), + INTSRC(5), + INTSRC(6), + INTSRC(7), + INTSRC(8), + INTSRC(9), + INTSRC(10), + INTSRC(11), + INTSRC(12), + INTSRC(13), + INTSRC(14), + INTSRC(15), + }; + + static void + atpic_enable_source(struct intsrc *isrc) + { + struct atpic_intsrc *ai = (struct atpic_intsrc *)isrc; + struct atpic *ap = (struct atpic *)isrc->is_pic; + + mtx_lock_spin(&icu_lock); + *ap->at_imen &= ~(1 << ai->at_irq); + outb(ap->at_ioaddr + ICU_IMR_OFFSET, *ap->at_imen); + mtx_unlock_spin(&icu_lock); + } + + static void + atpic_disable_source(struct intsrc *isrc) + { + struct atpic_intsrc *ai = (struct atpic_intsrc *)isrc; + struct atpic *ap = (struct atpic *)isrc->is_pic; + + mtx_lock_spin(&icu_lock); + *ap->at_imen |= (1 << ai->at_irq); + outb(ap->at_ioaddr + ICU_IMR_OFFSET, *ap->at_imen); + mtx_unlock_spin(&icu_lock); + } + + static void + atpic_eoi_master(struct intsrc *isrc) + { + + KASSERT(isrc->is_pic == &atpics[MASTER].at_pic, + ("%s: mismatched pic", __func__)); + #ifndef AUTO_EOI_1 + mtx_lock_spin(&icu_lock); + outb(atpics[MASTER].at_ioaddr, ICU_EOI); + mtx_unlock_spin(&icu_lock); + #endif + } + + static void + atpic_eoi_slave(struct intsrc *isrc) + { + + KASSERT(isrc->is_pic == &atpics[SLAVE].at_pic, + ("%s: mismatched pic", __func__)); + #ifndef AUTO_EOI_2 + mtx_lock_spin(&icu_lock); + outb(atpics[SLAVE].at_ioaddr, ICU_EOI); + #ifndef AUTO_EOI_1 + outb(atpics[MASTER].at_ioaddr, ICU_EOI); + #endif + mtx_unlock_spin(&icu_lock); + #endif + } + + static void + atpic_enable_intr(struct intsrc *isrc) + { + struct atpic_intsrc *ai = (struct atpic_intsrc *)isrc; + struct atpic *ap = (struct atpic *)isrc->is_pic; + register_t eflags; + + mtx_lock_spin(&icu_lock); + eflags = intr_disable(); + setidt(ap->at_intbase + ai->at_irq, ai->at_intr, SDT_SYS386IGT, + SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + intr_restore(eflags); + mtx_unlock_spin(&icu_lock); + } + + static int + atpic_vector(struct intsrc *isrc) + { + struct atpic_intsrc *ai = (struct atpic_intsrc *)isrc; + struct atpic *ap = (struct atpic *)isrc->is_pic; + + return (IRQ(ap, ai)); + } + + static int + atpic_source_pending(struct intsrc *isrc) + { + struct atpic_intsrc *ai = (struct atpic_intsrc *)isrc; + struct atpic *ap = (struct atpic *)isrc->is_pic; + + return (inb(ap->at_ioaddr) & (1 << ai->at_irq)); + } + + static void + atpic_resume(struct intsrc *isrc) + { + struct atpic_intsrc *ai = (struct atpic_intsrc *)isrc; + struct atpic *ap = (struct atpic *)isrc->is_pic; + + if (ai->at_irq == 0) + i8259_init(ap, ap == &atpics[SLAVE]); + } + + static void + i8259_init(struct atpic *pic, int slave) + { + int imr_addr; + + /* Reset the PIC and program with next four bytes. */ + mtx_lock_spin(&icu_lock); + #ifdef DEV_MCA + if (MCA_system) + outb(pic->at_ioaddr, 0x19); + else + #endif + outb(pic->at_ioaddr, 0x11); + imr_addr = pic->at_ioaddr + ICU_IMR_OFFSET; + + /* Start vector. */ + outb(imr_addr, pic->at_intbase); + + /* + * Setup slave links. For the master pic, indicate what line + * the slave is configured on. For the slave indicate + * which line on the master we are connected to. + */ + if (slave) + outb(imr_addr, ICU_SLAVEID); /* my slave id is 7 */ + else + outb(imr_addr, IRQ_SLAVE); /* slave on line 7 */ + + /* Set mode. */ + if (slave) + outb(imr_addr, SLAVE_MODE); + else + outb(imr_addr, MASTER_MODE); + + /* Set interrupt enable mask. */ + outb(imr_addr, *pic->at_imen); + + /* Reset is finished, default to IRR on read. */ + outb(pic->at_ioaddr, 0x0a); + + #ifndef PC98 + /* Set priority order to 3-7, 0-2 (com2 first). */ + if (!slave) + outb(pic->at_ioaddr, 0xc0 | (3 - 1)); + #endif + mtx_unlock_spin(&icu_lock); + } + + void + atpic_startup(void) + { + + /* Start off with all interrupts disabled. */ + imen = 0xffff; + i8259_init(&atpics[MASTER], 0); + i8259_init(&atpics[SLAVE], 1); + atpic_enable_source((struct intsrc *)&atintrs[ICU_SLAVEID]); + } + + static void + atpic_init(void *dummy __unused) + { + struct atpic_intsrc *ai; + int i; + + /* Loop through all interrupt sources and add them. */ + for (i = 0; i < sizeof(atintrs) / sizeof(struct atpic_intsrc); i++) { + if (i == ICU_SLAVEID) + continue; + ai = &atintrs[i]; + intr_register_source(&ai->at_intsrc); + } + } + SYSINIT(atpic_init, SI_SUB_INTR, SI_ORDER_SECOND + 1, atpic_init, NULL) + + void + atpic_sched_ithd(struct intrframe iframe) + { + struct intsrc *isrc; + + KASSERT((uint)iframe.if_vec < ICU_LEN, + ("unknown int %d\n", iframe.if_vec)); + isrc = &atintrs[iframe.if_vec].at_intsrc; + intr_execute_handlers(isrc, &iframe); + } + + /* + * Older stuff that is not cleaned up yet. + */ + + #ifdef DEV_ISA + /* + * Return a bitmap of the current interrupt requests. This is 8259-specific + * and is only suitable for use at probe time. + */ + intrmask_t + isa_irq_pending(void) + { + u_char irr1; + u_char irr2; + + irr1 = inb(IO_ICU1); + irr2 = inb(IO_ICU2); + return ((irr2 << 8) | irr1); + } + #endif + Index: i386/isa/bs/bsif.h =========================================================================== --- sys/i386/isa/bs/bsif.h 2003/10/30 14:48:07 #3 +++ i386/isa/bs/bsif.h 2003/10/30 14:48:07 @@ -124,7 +124,6 @@ #include #include #include -#include #endif /* __FreeBSD__ */ /*************************************************** Index: i386/isa/clock.c =========================================================================== --- sys/i386/isa/clock.c 2003/10/30 14:48:07 #32 +++ i386/isa/clock.c 2003/10/30 14:48:07 @@ -71,14 +71,12 @@ #include #include #include +#include #include #include -#ifdef APIC_IO -#include +#if defined(SMP) +#include #endif -#if defined(SMP) || defined(APIC_IO) -#include -#endif /* SMP || APIC_IO */ #include #include @@ -89,20 +87,10 @@ #endif #include -#include - #ifdef DEV_MCA #include #endif -#ifdef APIC_IO -#include -/* The interrupt triggered by the 8254 (timer) chip */ -int apic_8254_intr; -static u_long read_intr_count(int vec); -static void setup_8254_mixed_mode(void); -#endif - /* * 32-bit time_t's can't reach leap years before 1904 or after 2036, so we * can use a simple formula for leap years. @@ -150,6 +138,7 @@ static u_int32_t i8254_lastcount; static u_int32_t i8254_offset; static int i8254_ticked; +static struct intsrc *i8254_intsrc; #ifndef BURN_BRIDGES /* * XXX new_function and timer_func should not handle clockframes, but @@ -187,7 +176,7 @@ }; static void -clkintr(struct clockframe frame) +clkintr(struct clockframe *frame) { if (timecounter->tc_get_timecount == i8254_get_timecount) { @@ -201,7 +190,7 @@ clkintr_pending = 0; mtx_unlock_spin(&clock_lock); } - timer_func(&frame); + timer_func(frame); #ifdef SMP if (timer_func == hardclock) forward_hardclock(); @@ -216,7 +205,7 @@ if ((timer0_prescaler_count += timer0_max_count) >= hardclock_max_count) { timer0_prescaler_count -= hardclock_max_count; - hardclock(&frame); + hardclock(frame); #ifdef SMP forward_hardclock(); #endif @@ -251,7 +240,7 @@ timer0_prescaler_count = 0; timer_func = hardclock; timer0_state = RELEASED; - hardclock(&frame); + hardclock(frame); #ifdef SMP forward_hardclock(); #endif @@ -379,16 +368,16 @@ * in the statistics, but the stat clock will no longer stop. */ static void -rtcintr(struct clockframe frame) +rtcintr(struct clockframe *frame) { while (rtcin(RTC_INTR) & RTCIR_PERIOD) { if (profprocs != 0) { if (--pscnt == 0) pscnt = psdiv; - profclock(&frame); + profclock(frame); } if (pscnt == psdiv) - statclock(&frame); + statclock(frame); #ifdef SMP forward_statclock(); #endif @@ -931,11 +920,6 @@ cpu_initclocks() { int diag; -#ifdef APIC_IO - int apic_8254_trial; - void *clkdesc; -#endif /* APIC_IO */ - register_t crit; if (statclock_disable) { /* @@ -951,48 +935,10 @@ profhz = RTC_PROFRATE; } - /* Finish initializing 8253 timer 0. */ -#ifdef APIC_IO - - apic_8254_intr = isa_apic_irq(0); - apic_8254_trial = 0; - if (apic_8254_intr >= 0 ) { - if (apic_int_type(0, 0) == 3) - apic_8254_trial = 1; - } else { - /* look for ExtInt on pin 0 */ - if (apic_int_type(0, 0) == 3) { - apic_8254_intr = apic_irq(0, 0); - setup_8254_mixed_mode(); - } else - panic("APIC_IO: Cannot route 8254 interrupt to CPU"); - } - - inthand_add("clk", apic_8254_intr, (driver_intr_t *)clkintr, NULL, - INTR_TYPE_CLK | INTR_FAST, &clkdesc); - crit = intr_disable(); - mtx_lock_spin(&icu_lock); - INTREN(1 << apic_8254_intr); - mtx_unlock_spin(&icu_lock); - intr_restore(crit); - -#else /* APIC_IO */ - - /* - * XXX Check the priority of this interrupt handler. I - * couldn't find anything suitable in the BSD/OS code (grog, - * 19 July 2000). - */ - inthand_add("clk", 0, (driver_intr_t *)clkintr, NULL, + /* Finish initializing 8254 timer 0. */ + intr_add_handler("clk", 0, (driver_intr_t *)clkintr, NULL, INTR_TYPE_CLK | INTR_FAST, NULL); - crit = intr_disable(); - mtx_lock_spin(&icu_lock); - INTREN(IRQ0); - mtx_unlock_spin(&icu_lock); - intr_restore(crit); -#endif /* APIC_IO */ - /* Initialize RTC. */ writertc(RTC_STATUSA, rtc_statusa); writertc(RTC_STATUSB, RTCSB_24HR); @@ -1004,120 +950,15 @@ if (diag != 0) printf("RTC BIOS diagnostic error %b\n", diag, RTCDG_BITS); -#ifdef APIC_IO - if (isa_apic_irq(8) != 8) - panic("APIC RTC != 8"); -#endif /* APIC_IO */ - - inthand_add("rtc", 8, (driver_intr_t *)rtcintr, NULL, + intr_add_handler("rtc", 8, (driver_intr_t *)rtcintr, NULL, INTR_TYPE_CLK | INTR_FAST, NULL); + i8254_intsrc = intr_lookup_source(8); - crit = intr_disable(); - mtx_lock_spin(&icu_lock); -#ifdef APIC_IO - INTREN(APIC_IRQ8); -#else - INTREN(IRQ8); -#endif /* APIC_IO */ - mtx_unlock_spin(&icu_lock); - intr_restore(crit); - writertc(RTC_STATUSB, rtc_statusb); -#ifdef APIC_IO - if (apic_8254_trial) { - - printf("APIC_IO: Testing 8254 interrupt delivery\n"); - while (read_intr_count(8) < 6) - ; /* nothing */ - if (read_intr_count(apic_8254_intr) < 3) { - /* - * The MP table is broken. - * The 8254 was not connected to the specified pin - * on the IO APIC. - * Workaround: Limited variant of mixed mode. - */ - - crit = intr_disable(); - mtx_lock_spin(&icu_lock); - INTRDIS(1 << apic_8254_intr); - mtx_unlock_spin(&icu_lock); - intr_restore(crit); - inthand_remove(clkdesc); - printf("APIC_IO: Broken MP table detected: " - "8254 is not connected to " - "IOAPIC #%d intpin %d\n", - int_to_apicintpin[apic_8254_intr].ioapic, - int_to_apicintpin[apic_8254_intr].int_pin); - /* - * Revoke current ISA IRQ 0 assignment and - * configure a fallback interrupt routing from - * the 8254 Timer via the 8259 PIC to the - * an ExtInt interrupt line on IOAPIC #0 intpin 0. - * We reuse the low level interrupt handler number. - */ - if (apic_irq(0, 0) < 0) { - revoke_apic_irq(apic_8254_intr); - assign_apic_irq(0, 0, apic_8254_intr); - } - apic_8254_intr = apic_irq(0, 0); - setup_8254_mixed_mode(); - inthand_add("clk", apic_8254_intr, - (driver_intr_t *)clkintr, NULL, - INTR_TYPE_CLK | INTR_FAST, NULL); - crit = intr_disable(); - mtx_lock_spin(&icu_lock); - INTREN(1 << apic_8254_intr); - mtx_unlock_spin(&icu_lock); - intr_restore(crit); - } - - } - if (apic_int_type(0, 0) != 3 || - int_to_apicintpin[apic_8254_intr].ioapic != 0 || - int_to_apicintpin[apic_8254_intr].int_pin != 0) - printf("APIC_IO: routing 8254 via IOAPIC #%d intpin %d\n", - int_to_apicintpin[apic_8254_intr].ioapic, - int_to_apicintpin[apic_8254_intr].int_pin); - else - printf("APIC_IO: " - "routing 8254 via 8259 and IOAPIC #0 intpin 0\n"); -#endif - init_TSC_tc(); } -#ifdef APIC_IO -static u_long -read_intr_count(int vec) -{ - u_long *up; - up = intr_countp[vec]; - if (up) - return *up; - return 0UL; -} - -static void -setup_8254_mixed_mode() -{ - /* - * Allow 8254 timer to INTerrupt 8259: - * re-initialize master 8259: - * reset; prog 4 bytes, single ICU, edge triggered - */ - outb(IO_ICU1, 0x13); - outb(IO_ICU1 + 1, NRSVIDT); /* start vector (unused) */ - outb(IO_ICU1 + 1, 0x00); /* ignore slave */ - outb(IO_ICU1 + 1, 0x03); /* auto EOI, 8086 */ - outb(IO_ICU1 + 1, 0xfe); /* unmask INT0 */ - - /* program IO APIC for type 3 INT on INT0 */ - if (ext_int_setup(0, 0) < 0) - panic("8254 redirect via APIC pin0 impossible!"); -} -#endif - void cpu_startprofclock(void) { @@ -1181,14 +1022,8 @@ if (count < i8254_lastcount || (!i8254_ticked && (clkintr_pending || ((count < 20 || (!(eflags & PSL_I) && count < timer0_max_count / 2u)) && -#ifdef APIC_IO -#define lapic_irr1 ((volatile u_int *)&lapic)[0x210 / 4] /* XXX XXX */ - /* XXX this assumes that apic_8254_intr is < 24. */ - (lapic_irr1 & (1 << apic_8254_intr)))) -#else - (inb(IO_ICU1) & 1))) -#endif - )) { + i8254_intsrc != NULL && + i8254_intsrc->is_pic->pic_source_pending(i8254_intsrc))))) { i8254_ticked = 1; i8254_offset += timer0_max_count; } Index: i386/isa/icu.h =========================================================================== --- sys/i386/isa/icu.h 2003/10/30 14:48:07 #5 +++ i386/isa/icu.h 2003/10/30 14:48:07 @@ -45,39 +45,7 @@ #ifndef _I386_ISA_ICU_H_ #define _I386_ISA_ICU_H_ -#ifndef LOCORE - /* - * Note: - * Most of the SMP equivilants of the icu macros are coded - * elsewhere in an MP-safe fashion. - * In particular note that the 'imen' variable is opaque. - * DO NOT access imen directly, use INTREN()/INTRDIS(). - */ - -void INTREN(u_int); -void INTRDIS(u_int); - -#ifdef APIC_IO -extern unsigned apic_imen; /* APIC interrupt mask enable */ -#else -extern unsigned imen; /* interrupt mask enable */ -#endif - -#endif /* LOCORE */ - - -#ifdef APIC_IO -/* - * Note: The APIC uses different values for IRQxxx. - * Unfortunately many drivers use the 8259 values as indexes - * into tables, etc. The APIC equivilants are kept as APIC_IRQxxx. - * The 8259 versions have to be used in SMP for legacy operation - * of the drivers. - */ -#endif /* APIC_IO */ - -/* * Interrupt enable bits - in normal order of priority (which we change) */ #ifdef PC98 @@ -122,6 +90,7 @@ * Interrupt Control offset into Interrupt descriptor table (IDT) */ #define ICU_OFFSET 32 /* 0-31 are processor exceptions */ +#define ICU_LEN 16 /* 32-47 are ISA interrupts */ #ifdef PC98 #define ICU_IMR_OFFSET 2 @@ -130,23 +99,13 @@ #define ICU_IMR_OFFSET 1 #define ICU_SLAVEID 2 #endif + #define ICU_EOI 0x20 - -#ifdef APIC_IO - -/* 32-47: ISA IRQ0-IRQ15, 48-63: IO APIC IRQ16-IRQ31 */ -#define ICU_LEN 32 -#define HWI_MASK 0xffffffff /* bits for h/w interrupts */ -#define NHWI 32 - -#else - -#define ICU_LEN 16 /* 32-47 are ISA interrupts */ #define HWI_MASK 0xffff /* bits for h/w interrupts */ -#define NHWI 16 -#endif /* APIC_IO */ - -#define INTRCNT_COUNT (1 + ICU_LEN + 2 * ICU_LEN) +#ifndef LOCORE +void atpic_sched_ithd(struct intrframe iframe); +void atpic_startup(void); +#endif #endif /* !_I386_ISA_ICU_H_ */ Index: i386/isa/icu_vector.s =========================================================================== --- sys/i386/isa/icu_vector.s 2003/10/30 14:48:07 #10 +++ i386/isa/icu_vector.s 2003/10/30 14:48:07 @@ -1,10 +1,58 @@ -/* +/*- + * Copyright (c) 1989, 1990 William F. Jolitz. + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * * from: vector.s, 386BSD 0.1 unknown origin * $FreeBSD: src/sys/i386/isa/icu_vector.s,v 1.37 2002/04/06 08:25:05 nyan Exp $ */ +/* + * Interrupt entry points for external interrupts triggered by the 8259A + * master and slave interrupt controllers. + */ + +#include "opt_auto_eoi.h" + +#include +#include +#ifdef PC98 +#include +#else +#include +#endif + +#include "assym.s" + #define IRQ_BIT(irq_num) (1 << ((irq_num) % 8)) -#define IRQ_LBIT(irq_num) (1 << (irq_num)) #define IRQ_BYTE(irq_num) ((irq_num) >> 3) #ifdef AUTO_EOI_1 @@ -38,119 +86,8 @@ #endif -#define PUSH_FRAME \ - pushl $0 ; /* dummy error code */ \ - pushl $0 ; /* dummy trap type */ \ - pushal ; /* 8 ints */ \ - pushl %ds ; /* save data and extra segments ... */ \ - pushl %es ; \ - pushl %fs - -#define PUSH_DUMMY \ - pushfl ; /* eflags */ \ - pushl %cs ; /* cs */ \ - pushl 12(%esp) ; /* original caller eip */ \ - pushl $0 ; /* dummy error code */ \ - pushl $0 ; /* dummy trap type */ \ - subl $11*4,%esp - -#define POP_FRAME \ - popl %fs ; \ - popl %es ; \ - popl %ds ; \ - popal ; \ - addl $4+4,%esp - -#define POP_DUMMY \ - addl $16*4,%esp - -#define MASK_IRQ(icu, irq_num) \ - movb imen + IRQ_BYTE(irq_num),%al ; \ - orb $IRQ_BIT(irq_num),%al ; \ - movb %al,imen + IRQ_BYTE(irq_num) ; \ - outb %al,$icu+ICU_IMR_OFFSET - -#define UNMASK_IRQ(icu, irq_num) \ - movb imen + IRQ_BYTE(irq_num),%al ; \ - andb $~IRQ_BIT(irq_num),%al ; \ - movb %al,imen + IRQ_BYTE(irq_num) ; \ - outb %al,$icu+ICU_IMR_OFFSET /* * Macros for interrupt interrupt entry, call to handler, and exit. - */ - -#define FAST_INTR(irq_num, vec_name, icu, enable_icus) \ - .text ; \ - SUPERALIGN_TEXT ; \ -IDTVEC(vec_name) ; \ - PUSH_FRAME ; \ - mov $KDSEL,%ax ; \ - mov %ax,%ds ; \ - mov %ax,%es ; \ - mov $KPSEL,%ax ; \ - mov %ax,%fs ; \ - FAKE_MCOUNT((12+ACTUALLY_PUSHED)*4(%esp)) ; \ - movl PCPU(CURTHREAD),%ebx ; \ - cmpl $0,TD_CRITNEST(%ebx) ; \ - je 1f ; \ -; \ - movl $1,PCPU(INT_PENDING) ; \ - orl $IRQ_LBIT(irq_num),PCPU(FPENDING) ; \ - MASK_IRQ(icu, irq_num) ; \ - enable_icus ; \ - jmp 10f ; \ -1: ; \ - incl TD_CRITNEST(%ebx) ; \ - incl TD_INTR_NESTING_LEVEL(%ebx) ; \ - pushl intr_unit + (irq_num) * 4 ; \ - call *intr_handler + (irq_num) * 4 ; \ - addl $4,%esp ; \ - enable_icus ; \ - incl cnt+V_INTR ; /* book-keeping can wait */ \ - movl intr_countp + (irq_num) * 4,%eax ; \ - incl (%eax) ; \ - decl TD_CRITNEST(%ebx) ; \ - cmpl $0,PCPU(INT_PENDING) ; \ - je 2f ; \ -; \ - call i386_unpend ; \ -2: ; \ - decl TD_INTR_NESTING_LEVEL(%ebx) ; \ -10: ; \ - MEXITCOUNT ; \ - jmp doreti - -/* - * Restart a fast interrupt that was held up by a critical section. - * This routine is called from unpend(). unpend() ensures we are - * in a critical section and deals with the interrupt nesting level - * for us. If we previously masked the irq, we have to unmask it. - * - * We have a choice. We can regenerate the irq using the 'int' - * instruction or we can create a dummy frame and call the interrupt - * handler directly. I've chosen to use the dummy-frame method. - */ -#define FAST_UNPEND(irq_num, vec_name, icu) \ - .text ; \ - SUPERALIGN_TEXT ; \ -IDTVEC(vec_name) ; \ -; \ - pushl %ebp ; \ - movl %esp, %ebp ; \ - PUSH_DUMMY ; \ - pushl intr_unit + (irq_num) * 4 ; \ - call *intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ - addl $4, %esp ; \ - incl cnt+V_INTR ; /* book-keeping can wait */ \ - movl intr_countp + (irq_num) * 4,%eax ; \ - incl (%eax) ; \ - UNMASK_IRQ(icu, irq_num) ; \ - POP_DUMMY ; \ - popl %ebp ; \ - ret - -/* - * Slow, threaded interrupts. * * XXX Most of the parameters here are obsolete. Fix this when we're * done. @@ -162,7 +99,12 @@ .text ; \ SUPERALIGN_TEXT ; \ IDTVEC(vec_name) ; \ - PUSH_FRAME ; \ + pushl $0 ; /* dummy error code */ \ + pushl $0 ; /* dummy trap type */ \ + pushal ; /* 8 ints */ \ + pushl %ds ; /* save data and extra segments ... */ \ + pushl %es ; \ + pushl %fs ; \ mov $KDSEL,%ax ; /* load kernel ds, es and fs */ \ mov %ax,%ds ; \ mov %ax,%es ; \ @@ -170,84 +112,41 @@ mov %ax,%fs ; \ ; \ maybe_extra_ipending ; \ - MASK_IRQ(icu, irq_num) ; \ + movb imen + IRQ_BYTE(irq_num),%al ; \ + orb $IRQ_BIT(irq_num),%al ; \ + movb %al,imen + IRQ_BYTE(irq_num) ; \ + outb %al,$icu+ICU_IMR_OFFSET ; \ enable_icus ; \ ; \ movl PCPU(CURTHREAD),%ebx ; \ - cmpl $0,TD_CRITNEST(%ebx) ; \ - je 1f ; \ - movl $1,PCPU(INT_PENDING); \ - orl $IRQ_LBIT(irq_num),PCPU(IPENDING) ; \ - jmp 10f ; \ -1: ; \ incl TD_INTR_NESTING_LEVEL(%ebx) ; \ ; \ FAKE_MCOUNT(13*4(%esp)) ; /* XXX late to avoid double count */ \ - cmpl $0,PCPU(INT_PENDING) ; \ - je 9f ; \ - call i386_unpend ; \ -9: ; \ pushl $irq_num; /* pass the IRQ */ \ - call sched_ithd ; \ + call atpic_sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ ; \ decl TD_INTR_NESTING_LEVEL(%ebx) ; \ -10: ; \ MEXITCOUNT ; \ jmp doreti MCOUNT_LABEL(bintr) - FAST_INTR(0,fastintr0, IO_ICU1, ENABLE_ICU1) - FAST_INTR(1,fastintr1, IO_ICU1, ENABLE_ICU1) - FAST_INTR(2,fastintr2, IO_ICU1, ENABLE_ICU1) - FAST_INTR(3,fastintr3, IO_ICU1, ENABLE_ICU1) - FAST_INTR(4,fastintr4, IO_ICU1, ENABLE_ICU1) - FAST_INTR(5,fastintr5, IO_ICU1, ENABLE_ICU1) - FAST_INTR(6,fastintr6, IO_ICU1, ENABLE_ICU1) - FAST_INTR(7,fastintr7, IO_ICU1, ENABLE_ICU1) - FAST_INTR(8,fastintr8, IO_ICU2, ENABLE_ICU1_AND_2) - FAST_INTR(9,fastintr9, IO_ICU2, ENABLE_ICU1_AND_2) - FAST_INTR(10,fastintr10, IO_ICU2, ENABLE_ICU1_AND_2) - FAST_INTR(11,fastintr11, IO_ICU2, ENABLE_ICU1_AND_2) - FAST_INTR(12,fastintr12, IO_ICU2, ENABLE_ICU1_AND_2) - FAST_INTR(13,fastintr13, IO_ICU2, ENABLE_ICU1_AND_2) - FAST_INTR(14,fastintr14, IO_ICU2, ENABLE_ICU1_AND_2) - FAST_INTR(15,fastintr15, IO_ICU2, ENABLE_ICU1_AND_2) - #define CLKINTR_PENDING movl $1,CNAME(clkintr_pending) /* Threaded interrupts */ - INTR(0,intr0, IO_ICU1, ENABLE_ICU1, CLKINTR_PENDING) - INTR(1,intr1, IO_ICU1, ENABLE_ICU1,) - INTR(2,intr2, IO_ICU1, ENABLE_ICU1,) - INTR(3,intr3, IO_ICU1, ENABLE_ICU1,) - INTR(4,intr4, IO_ICU1, ENABLE_ICU1,) - INTR(5,intr5, IO_ICU1, ENABLE_ICU1,) - INTR(6,intr6, IO_ICU1, ENABLE_ICU1,) - INTR(7,intr7, IO_ICU1, ENABLE_ICU1,) - INTR(8,intr8, IO_ICU2, ENABLE_ICU1_AND_2,) - INTR(9,intr9, IO_ICU2, ENABLE_ICU1_AND_2,) - INTR(10,intr10, IO_ICU2, ENABLE_ICU1_AND_2,) - INTR(11,intr11, IO_ICU2, ENABLE_ICU1_AND_2,) - INTR(12,intr12, IO_ICU2, ENABLE_ICU1_AND_2,) - INTR(13,intr13, IO_ICU2, ENABLE_ICU1_AND_2,) - INTR(14,intr14, IO_ICU2, ENABLE_ICU1_AND_2,) - INTR(15,intr15, IO_ICU2, ENABLE_ICU1_AND_2,) - - FAST_UNPEND(0,fastunpend0, IO_ICU1) - FAST_UNPEND(1,fastunpend1, IO_ICU1) - FAST_UNPEND(2,fastunpend2, IO_ICU1) - FAST_UNPEND(3,fastunpend3, IO_ICU1) - FAST_UNPEND(4,fastunpend4, IO_ICU1) - FAST_UNPEND(5,fastunpend5, IO_ICU1) - FAST_UNPEND(6,fastunpend6, IO_ICU1) - FAST_UNPEND(7,fastunpend7, IO_ICU1) - FAST_UNPEND(8,fastunpend8, IO_ICU2) - FAST_UNPEND(9,fastunpend9, IO_ICU2) - FAST_UNPEND(10,fastunpend10, IO_ICU2) - FAST_UNPEND(11,fastunpend11, IO_ICU2) - FAST_UNPEND(12,fastunpend12, IO_ICU2) - FAST_UNPEND(13,fastunpend13, IO_ICU2) - FAST_UNPEND(14,fastunpend14, IO_ICU2) - FAST_UNPEND(15,fastunpend15, IO_ICU2) + INTR(0,atpic_intr0, IO_ICU1, ENABLE_ICU1, CLKINTR_PENDING) + INTR(1,atpic_intr1, IO_ICU1, ENABLE_ICU1,) + INTR(2,atpic_intr2, IO_ICU1, ENABLE_ICU1,) + INTR(3,atpic_intr3, IO_ICU1, ENABLE_ICU1,) + INTR(4,atpic_intr4, IO_ICU1, ENABLE_ICU1,) + INTR(5,atpic_intr5, IO_ICU1, ENABLE_ICU1,) + INTR(6,atpic_intr6, IO_ICU1, ENABLE_ICU1,) + INTR(7,atpic_intr7, IO_ICU1, ENABLE_ICU1,) + INTR(8,atpic_intr8, IO_ICU2, ENABLE_ICU1_AND_2,) + INTR(9,atpic_intr9, IO_ICU2, ENABLE_ICU1_AND_2,) + INTR(10,atpic_intr10, IO_ICU2, ENABLE_ICU1_AND_2,) + INTR(11,atpic_intr11, IO_ICU2, ENABLE_ICU1_AND_2,) + INTR(12,atpic_intr12, IO_ICU2, ENABLE_ICU1_AND_2,) + INTR(13,atpic_intr13, IO_ICU2, ENABLE_ICU1_AND_2,) + INTR(14,atpic_intr14, IO_ICU2, ENABLE_ICU1_AND_2,) + INTR(15,atpic_intr15, IO_ICU2, ENABLE_ICU1_AND_2,) MCOUNT_LABEL(eintr) - Index: i386/isa/if_rdp.c =========================================================================== --- sys/i386/isa/if_rdp.c 2003/10/30 14:48:07 #12 +++ i386/isa/if_rdp.c 2003/10/30 14:48:07 @@ -91,7 +91,6 @@ #include #include #include -#include #ifndef COMPAT_OLDISA #error "The rdp device requires the old isa compatibility shims" Index: i386/isa/intr_machdep.c =========================================================================== --- sys/i386/isa/intr_machdep.c 2003/10/30 14:48:07 #21 +++ i386/isa/intr_machdep.c 2003/10/30 14:48:07 @@ -39,127 +39,18 @@ #include __FBSDID("$FreeBSD: src/sys/i386/isa/intr_machdep.c,v 1.76 2003/06/02 16:32:54 obrien Exp $"); -#include "opt_auto_eoi.h" -#include "opt_isa.h" #include "opt_mca.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include #include -#include #include -#include - -#if defined(APIC_IO) -#include /** FAST_HI */ -#include -#include -#endif /* APIC_IO */ -#ifdef PC98 -#include -#include -#include -#else -#include -#endif -#include - -#ifdef DEV_ISA -#include -#endif -#include -#include -#ifdef APIC_IO -#include -#endif #ifdef DEV_MCA #include #endif -/* - * Per-interrupt data. - */ -u_long *intr_countp[ICU_LEN]; /* pointers to interrupt counters */ -driver_intr_t *intr_handler[ICU_LEN]; /* first level interrupt handler */ -struct ithd *ithds[ICU_LEN]; /* real interrupt handler */ -void *intr_unit[ICU_LEN]; - -static struct mtx ithds_table_lock; /* protect the ithds table */ - -static inthand_t *fastintr[ICU_LEN] = { - IDTVEC(fastintr0), IDTVEC(fastintr1), - IDTVEC(fastintr2), IDTVEC(fastintr3), - IDTVEC(fastintr4), IDTVEC(fastintr5), - IDTVEC(fastintr6), IDTVEC(fastintr7), - IDTVEC(fastintr8), IDTVEC(fastintr9), - IDTVEC(fastintr10), IDTVEC(fastintr11), - IDTVEC(fastintr12), IDTVEC(fastintr13), - IDTVEC(fastintr14), IDTVEC(fastintr15), -#if defined(APIC_IO) - IDTVEC(fastintr16), IDTVEC(fastintr17), - IDTVEC(fastintr18), IDTVEC(fastintr19), - IDTVEC(fastintr20), IDTVEC(fastintr21), - IDTVEC(fastintr22), IDTVEC(fastintr23), - IDTVEC(fastintr24), IDTVEC(fastintr25), - IDTVEC(fastintr26), IDTVEC(fastintr27), - IDTVEC(fastintr28), IDTVEC(fastintr29), - IDTVEC(fastintr30), IDTVEC(fastintr31), -#endif /* APIC_IO */ -}; - -static unpendhand_t *fastunpend[ICU_LEN] = { - IDTVEC(fastunpend0), IDTVEC(fastunpend1), - IDTVEC(fastunpend2), IDTVEC(fastunpend3), - IDTVEC(fastunpend4), IDTVEC(fastunpend5), - IDTVEC(fastunpend6), IDTVEC(fastunpend7), - IDTVEC(fastunpend8), IDTVEC(fastunpend9), - IDTVEC(fastunpend10), IDTVEC(fastunpend11), - IDTVEC(fastunpend12), IDTVEC(fastunpend13), - IDTVEC(fastunpend14), IDTVEC(fastunpend15), -#if defined(APIC_IO) - IDTVEC(fastunpend16), IDTVEC(fastunpend17), - IDTVEC(fastunpend18), IDTVEC(fastunpend19), - IDTVEC(fastunpend20), IDTVEC(fastunpend21), - IDTVEC(fastunpend22), IDTVEC(fastunpend23), - IDTVEC(fastunpend24), IDTVEC(fastunpend25), - IDTVEC(fastunpend26), IDTVEC(fastunpend27), - IDTVEC(fastunpend28), IDTVEC(fastunpend29), - IDTVEC(fastunpend30), IDTVEC(fastunpend31), -#endif /* APIC_IO */ -}; - -static inthand_t *slowintr[ICU_LEN] = { - IDTVEC(intr0), IDTVEC(intr1), IDTVEC(intr2), IDTVEC(intr3), - IDTVEC(intr4), IDTVEC(intr5), IDTVEC(intr6), IDTVEC(intr7), - IDTVEC(intr8), IDTVEC(intr9), IDTVEC(intr10), IDTVEC(intr11), - IDTVEC(intr12), IDTVEC(intr13), IDTVEC(intr14), IDTVEC(intr15), -#if defined(APIC_IO) - IDTVEC(intr16), IDTVEC(intr17), IDTVEC(intr18), IDTVEC(intr19), - IDTVEC(intr20), IDTVEC(intr21), IDTVEC(intr22), IDTVEC(intr23), - IDTVEC(intr24), IDTVEC(intr25), IDTVEC(intr26), IDTVEC(intr27), - IDTVEC(intr28), IDTVEC(intr29), IDTVEC(intr30), IDTVEC(intr31), -#endif /* APIC_IO */ -}; - -static driver_intr_t isa_strayintr; - -static void ithds_init(void *dummy); -static void ithread_enable(int vector); -static void ithread_disable(int vector); -static void init_i8259(void); - #ifdef PC98 #define NMI_PARITY 0x04 #define NMI_EPARITY 0x02 @@ -171,80 +62,12 @@ #define ENMI_IOSTATUS (1 << 5) #endif -#ifdef DEV_ISA -/* - * Bus attachment for the ISA PIC. - */ -static struct isa_pnp_id atpic_ids[] = { - { 0x0000d041 /* PNP0000 */, "AT interrupt controller" }, - { 0 } -}; - -static int -atpic_probe(device_t dev) -{ - int result; - - if ((result = ISA_PNP_PROBE(device_get_parent(dev), dev, atpic_ids)) <= 0) - device_quiet(dev); - return(result); -} - -/* - * In the APIC_IO case we might be granted IRQ 2, as this is typically - * consumed by chaining between the two PIC components. If we're using - * the APIC, however, this may not be the case, and as such we should - * free the resource. (XXX untested) - * - * The generic ISA attachment code will handle allocating any other resources - * that we don't explicitly claim here. - */ -static int -atpic_attach(device_t dev) -{ -#ifdef APIC_IO - int rid; - struct resource *res; - - /* try to allocate our IRQ and then free it */ - rid = 0; - res = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, 0, ~0, 1, 0); - if (res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, rid, res); -#endif - return(0); -} - -static device_method_t atpic_methods[] = { - /* Device interface */ - DEVMETHOD(device_probe, atpic_probe), - DEVMETHOD(device_attach, atpic_attach), - DEVMETHOD(device_detach, bus_generic_detach), - DEVMETHOD(device_shutdown, bus_generic_shutdown), - DEVMETHOD(device_suspend, bus_generic_suspend), - DEVMETHOD(device_resume, bus_generic_resume), - { 0, 0 } -}; - -static driver_t atpic_driver = { - "atpic", - atpic_methods, - 1, /* no softc */ -}; - -static devclass_t atpic_devclass; - -DRIVER_MODULE(atpic, isa, atpic_driver, atpic_devclass, 0, 0); -DRIVER_MODULE(atpic, acpi, atpic_driver, atpic_devclass, 0, 0); -#endif /* DEV_ISA */ - /* * Handle a NMI, possibly a machine check. * return true to panic system, false to ignore. */ int -isa_nmi(cd) - int cd; +isa_nmi(int cd) { int retval = 0; #ifdef PC98 @@ -307,410 +130,3 @@ #endif return(retval); } - -/* - * ICU reinitialize when ICU configuration has lost. - */ -void -icu_reinit() -{ - int i; - register_t crit; - - crit = intr_disable(); - mtx_lock_spin(&icu_lock); - init_i8259(); - for(i=0;i= sizeof(buf)) - goto use_bitbucket; - - /* - * Search for `buf' in `intrnames'. In the usual case when it is - * not found, append it to the end if there is enough space (the \0 - * terminator for the previous string, if any, becomes a separator). - */ - for (cp = intrnames, name_index = 0; - cp != eintrnames && name_index < NR_INTRNAMES; - cp += strlen(cp) + 1, name_index++) { - if (*cp == '\0') { - if (strlen(buf) >= eintrnames - cp) - break; - strcpy(cp, buf); - goto found; - } - if (strcmp(cp, buf) == 0) - goto found; - } - -use_bitbucket: - printf("update_intrname: counting %s irq%d as %s\n", name, intr, - intrnames); - name_index = 0; -found: - intr_countp[intr] = &intrcnt[name_index]; -} - -int -icu_setup(int intr, driver_intr_t *handler, void *arg, int flags) -{ -#ifdef FAST_HI - int select; /* the select register is 8 bits */ - int vector; - u_int32_t value; /* the window register is 32 bits */ -#endif /* FAST_HI */ - register_t crit; - -#if defined(APIC_IO) - if ((u_int)intr >= ICU_LEN) /* no 8259 SLAVE to ignore */ -#else - if ((u_int)intr >= ICU_LEN || intr == ICU_SLAVEID) -#endif /* APIC_IO */ - return (EINVAL); -#if 0 - if (intr_handler[intr] != isa_strayintr) - return (EBUSY); -#endif - - crit = intr_disable(); - mtx_lock_spin(&icu_lock); - intr_handler[intr] = handler; - intr_unit[intr] = arg; -#ifdef FAST_HI - if (flags & INTR_FAST) { - vector = TPR_FAST_INTS + intr; - setidt(vector, fastintr[intr], - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - } - else { - vector = TPR_SLOW_INTS + intr; - setidt(vector, slowintr[intr], - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - } -#ifdef APIC_INTR_REORDER - set_lapic_isrloc(intr, vector); -#endif - /* - * Reprogram the vector in the IO APIC. - */ - if (int_to_apicintpin[intr].ioapic >= 0) { - select = int_to_apicintpin[intr].redirindex; - value = io_apic_read(int_to_apicintpin[intr].ioapic, - select) & ~IOART_INTVEC; - io_apic_write(int_to_apicintpin[intr].ioapic, - select, value | vector); - } -#else - setidt(ICU_OFFSET + intr, - flags & INTR_FAST ? fastintr[intr] : slowintr[intr], - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); -#endif /* FAST_HI */ - INTREN(1 << intr); - mtx_unlock_spin(&icu_lock); - intr_restore(crit); - return (0); -} - -/* - * Dissociate an interrupt handler from an IRQ and set the handler to - * the stray interrupt handler. The 'handler' parameter is used only - * for consistency checking. - */ -int -icu_unset(intr, handler) - int intr; - driver_intr_t *handler; -{ - register_t crit; - - if ((u_int)intr >= ICU_LEN || handler != intr_handler[intr]) - return (EINVAL); - - crit = intr_disable(); - mtx_lock_spin(&icu_lock); - INTRDIS(1 << intr); - intr_countp[intr] = &intrcnt[1 + intr]; - intr_handler[intr] = isa_strayintr; - intr_unit[intr] = &intr_unit[intr]; -#ifdef FAST_HI_XXX - /* XXX how do I re-create dvp here? */ - setidt(flags & INTR_FAST ? TPR_FAST_INTS + intr : TPR_SLOW_INTS + intr, - slowintr[intr], SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); -#else /* FAST_HI */ -#ifdef APIC_INTR_REORDER - set_lapic_isrloc(intr, ICU_OFFSET + intr); -#endif - setidt(ICU_OFFSET + intr, slowintr[intr], SDT_SYS386IGT, SEL_KPL, - GSEL(GCODE_SEL, SEL_KPL)); -#endif /* FAST_HI */ - mtx_unlock_spin(&icu_lock); - intr_restore(crit); - return (0); -} - -static void -ithds_init(void *dummy) -{ - - mtx_init(&ithds_table_lock, "ithread table lock", NULL, MTX_SPIN); -} -SYSINIT(ithds_init, SI_SUB_INTR, SI_ORDER_SECOND, ithds_init, NULL); - -static void -ithread_enable(int vector) -{ - register_t crit; - - crit = intr_disable(); - mtx_lock_spin(&icu_lock); - INTREN(1 << vector); - mtx_unlock_spin(&icu_lock); - intr_restore(crit); -} - -static void -ithread_disable(int vector) -{ - register_t crit; - - crit = intr_disable(); - mtx_lock_spin(&icu_lock); - INTRDIS(1 << vector); - mtx_unlock_spin(&icu_lock); - intr_restore(crit); -} - -int -inthand_add(const char *name, int irq, driver_intr_t handler, void *arg, - enum intr_type flags, void **cookiep) -{ - struct ithd *ithd; /* descriptor for the IRQ */ - int errcode = 0; - int created_ithd = 0; - - /* - * Work around a race where more than one CPU may be registering - * handlers on the same IRQ at the same time. - */ - mtx_lock_spin(&ithds_table_lock); - ithd = ithds[irq]; - mtx_unlock_spin(&ithds_table_lock); - if (ithd == NULL) { - errcode = ithread_create(&ithd, irq, 0, ithread_disable, - ithread_enable, "irq%d:", irq); - if (errcode) - return (errcode); - mtx_lock_spin(&ithds_table_lock); - if (ithds[irq] == NULL) { - ithds[irq] = ithd; - created_ithd++; - mtx_unlock_spin(&ithds_table_lock); - } else { - struct ithd *orphan; - - orphan = ithd; - ithd = ithds[irq]; - mtx_unlock_spin(&ithds_table_lock); - ithread_destroy(orphan); - } - } - - errcode = ithread_add_handler(ithd, name, handler, arg, - ithread_priority(flags), flags, cookiep); - - if ((flags & INTR_FAST) == 0 || errcode) - /* - * The interrupt process must be in place, but - * not necessarily schedulable, before we - * initialize the ICU, since it may cause an - * immediate interrupt. - */ - if (icu_setup(irq, sched_ithd, arg, flags) != 0) - panic("inthand_add: Can't initialize ICU"); - - if (errcode) - return (errcode); - - if (flags & INTR_FAST) { - errcode = icu_setup(irq, handler, arg, flags); - if (errcode && bootverbose) - printf("\tinthand_add(irq%d) failed, result=%d\n", - irq, errcode); - if (errcode) - return (errcode); - } - - update_intrname(irq, name); - return (0); -} - -/* - * Deactivate and remove linked list the interrupt handler descriptor - * data connected created by an earlier call of inthand_add(), then - * adjust the interrupt masks if necessary. - * - * Return the memory held by the interrupt handler descriptor data - * structure to the system. First ensure the handler is not actively - * in use. - */ -int -inthand_remove(void *cookie) -{ - - return (ithread_remove_handler(cookie)); -} - -void -call_fast_unpend(int irq) -{ - fastunpend[irq](); -} - Index: i386/isa/npx.c =========================================================================== --- sys/i386/isa/npx.c 2003/10/30 14:48:07 #38 +++ i386/isa/npx.c 2003/10/30 14:48:07 @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -61,31 +62,25 @@ #include #include -#ifndef SMP #include -#endif #include #include #include #include #include -#ifndef SMP #include -#endif #include #include #include #include -#ifndef SMP #include #ifdef PC98 #include #else #include #endif -#endif -#include +#include #ifdef DEV_ISA #include #endif @@ -165,9 +160,7 @@ static void fpurstor(union savefpu *); static int npx_attach(device_t dev); static void npx_identify(driver_t *driver, device_t parent); -#ifndef SMP static void npx_intr(void *); -#endif static int npx_probe(device_t dev); #ifdef I586_CPU_XXX static long timezero(const char *funcname, @@ -180,10 +173,8 @@ CTLFLAG_RD, &hw_float, 0, "Floatingpoint instructions executed in hardware"); -#ifndef SMP static volatile u_int npx_intrs_while_probing; static volatile u_int npx_traps_while_probing; -#endif static union savefpu npx_cleanstate; static bool_t npx_cleanstate_ready; @@ -191,7 +182,6 @@ static bool_t npx_exists; static bool_t npx_irq13; -#ifndef SMP alias_for_inthand_t probetrap; __asm(" \n\ .text \n\ @@ -203,7 +193,6 @@ fnclex \n\ iret \n\ "); -#endif /* SMP */ /* * Identify routine. Create a connection point on our parent for probing. @@ -220,7 +209,6 @@ panic("npx_identify"); } -#ifndef SMP /* * Do minimal handling of npx interrupts to convert them to traps. */ @@ -230,9 +218,7 @@ { struct thread *td; -#ifndef SMP npx_intrs_while_probing++; -#endif /* * The BUSY# latch must be cleared in all cases so that the next @@ -264,7 +250,6 @@ mtx_unlock_spin(&sched_lock); } } -#endif /* !SMP */ /* * Probe routine. Initialize cr0 to give correct behaviour for [f]wait @@ -276,7 +261,6 @@ npx_probe(dev) device_t dev; { -#ifndef SMP struct gate_descriptor save_idt_npxtrap; struct resource *ioport_res, *irq_res; void *irq_cookie; @@ -307,7 +291,6 @@ if (bus_setup_intr(dev, irq_res, INTR_TYPE_MISC | INTR_FAST, npx_intr, NULL, &irq_cookie) != 0) panic("npx: can't create intr"); -#endif /* !SMP */ /* * Partially reset the coprocessor, if any. Some BIOS's don't reset @@ -348,16 +331,6 @@ device_set_desc(dev, "math processor"); -#ifdef SMP - - /* - * Exception 16 MUST work for SMP. - */ - npx_ex16 = hw_float = npx_exists = 1; - return (0); - -#else /* !SMP */ - /* * Don't use fwait here because it might hang. * Don't use fnop here because it usually hangs if there is no FPU. @@ -413,6 +386,10 @@ */ npx_irq13 = 1; idt[IDT_MF] = save_idt_npxtrap; +#ifdef SMP + if (mp_ncpus > 1) + panic("npx0 cannot use IRQ 13 on an SMP system"); +#endif return (0); } /* @@ -425,6 +402,10 @@ * emulator and say that it has been installed. XXX handle devices * that aren't really devices better. */ +#ifdef SMP + if (mp_ncpus > 1) + panic("npx0 cannot be emulated on an SMP system"); +#endif /* FALLTHROUGH */ no_irq13: idt[IDT_MF] = save_idt_npxtrap; @@ -435,20 +416,15 @@ * irq active then we would get it instead of exception 16. */ { - register_t crit; + struct intsrc *isrc; - crit = intr_disable(); - mtx_lock_spin(&icu_lock); - INTRDIS(1 << irq_num); - mtx_unlock_spin(&icu_lock); - intr_restore(crit); + isrc = intr_lookup_source(irq_num); + isrc->is_pic->pic_disable_source(isrc); } bus_release_resource(dev, SYS_RES_IRQ, irq_rid, irq_res); bus_release_resource(dev, SYS_RES_IOPORT, ioport_rid, ioport_res); return (0); - -#endif /* SMP */ } /* Index: i386/pci/pci_bus.c =========================================================================== --- sys/i386/pci/pci_bus.c 2003/10/30 14:48:07 #31 +++ i386/pci/pci_bus.c 2003/10/30 14:48:07 @@ -420,7 +420,7 @@ if (pci_cfgregopen() == 0) return ENXIO; - return 0; + return -100; } int Index: i386/pci/pci_cfgreg.c =========================================================================== --- sys/i386/pci/pci_cfgreg.c 2003/10/30 14:48:07 #31 +++ i386/pci/pci_cfgreg.c 2003/10/30 14:48:07 @@ -48,10 +48,6 @@ #include #include -#ifdef APIC_IO -#include -#endif /* APIC_IO */ - #include "pcib_if.h" #define PRVERB(a) do { \ @@ -201,60 +197,17 @@ pci_cfgregread(int bus, int slot, int func, int reg, int bytes) { uint32_t line; -#ifdef APIC_IO - uint32_t pin; /* - * If we are using the APIC, the contents of the intline - * register will probably be wrong (since they are set up for - * use with the PIC. Rather than rewrite these registers - * (maybe that would be smarter) we trap attempts to read them - * and translate to our private vector numbers. - */ - if ((reg == PCIR_INTLINE) && (bytes == 1)) { - - pin = pcireg_cfgread(bus, slot, func, PCIR_INTPIN, 1); - line = pcireg_cfgread(bus, slot, func, PCIR_INTLINE, 1); - - if (pin != 0) { - int airq; - - airq = pci_apic_irq(bus, slot, pin); - if (airq >= 0) { - /* PCI specific entry found in MP table */ - if (airq != line) - undirect_pci_irq(line); - return(airq); - } else { - /* - * PCI interrupts might be redirected - * to the ISA bus according to some MP - * tables. Use the same methods as - * used by the ISA devices devices to - * find the proper IOAPIC int pin. - */ - airq = isa_apic_irq(line); - if ((airq >= 0) && (airq != line)) { - /* XXX: undirect_pci_irq() ? */ - undirect_isa_irq(line); - return(airq); - } - } - } - return(line); - } -#else - /* * Some BIOS writers seem to want to ignore the spec and put * 0 in the intline rather than 255 to indicate none. The rest of * the code uses 255 as an invalid IRQ. */ if (reg == PCIR_INTLINE && bytes == 1) { line = pcireg_cfgread(bus, slot, func, PCIR_INTLINE, 1); - return pci_i386_map_intline(line); + return (pci_i386_map_intline(line)); } -#endif /* APIC_IO */ - return(pcireg_cfgread(bus, slot, func, reg, bytes)); + return (pcireg_cfgread(bus, slot, func, reg, bytes)); } /* Index: kern/kern_intr.c =========================================================================== --- sys/kern/kern_intr.c 2003/10/30 14:48:07 #39 +++ kern/kern_intr.c 2003/10/30 14:48:07 @@ -250,8 +250,6 @@ if (ithread == NULL || name == NULL || handler == NULL) return (EINVAL); - if ((flags & INTR_FAST) !=0) - flags |= INTR_EXCL; ih = malloc(sizeof(struct intrhand), M_ITHREAD, M_WAITOK | M_ZERO); ih->ih_handler = handler; @@ -260,7 +258,7 @@ ih->ih_ithread = ithread; ih->ih_pri = pri; if (flags & INTR_FAST) - ih->ih_flags = IH_FAST | IH_EXCLUSIVE; + ih->ih_flags = IH_FAST; else if (flags & INTR_EXCL) ih->ih_flags = IH_EXCLUSIVE; if (flags & INTR_MPSAFE) @@ -269,11 +267,17 @@ ih->ih_flags |= IH_ENTROPY; mtx_lock(&ithread->it_lock); - if ((flags & INTR_EXCL) !=0 && !TAILQ_EMPTY(&ithread->it_handlers)) + if ((flags & INTR_EXCL) != 0 && !TAILQ_EMPTY(&ithread->it_handlers)) goto fail; - if (!TAILQ_EMPTY(&ithread->it_handlers) && - (TAILQ_FIRST(&ithread->it_handlers)->ih_flags & IH_EXCLUSIVE) != 0) - goto fail; + if (!TAILQ_EMPTY(&ithread->it_handlers)) { + temp_ih = TAILQ_FIRST(&ithread->it_handlers); + if (temp_ih->ih_flags & IH_EXCLUSIVE) + goto fail; + if ((ih->ih_flags & IH_FAST) && !(temp_ih->ih_flags & IH_FAST)) + goto fail; + if (!(ih->ih_flags & IH_FAST) && (temp_ih->ih_flags & IH_FAST)) + goto fail; + } TAILQ_FOREACH(temp_ih, &ithread->it_handlers, ih_next) if (temp_ih->ih_pri > ih->ih_pri) Index: kern/subr_witness.c =========================================================================== --- sys/kern/subr_witness.c 2003/10/30 14:48:07 #81 +++ kern/subr_witness.c 2003/10/30 14:48:07 @@ -285,6 +285,7 @@ { "ng_node", &lock_class_mtx_spin }, { "ng_worklist", &lock_class_mtx_spin }, { "taskqueue_fast", &lock_class_mtx_spin }, + { "intr table", &lock_class_mtx_spin }, { "ithread table lock", &lock_class_mtx_spin }, { "sched lock", &lock_class_mtx_spin }, { "callout", &lock_class_mtx_spin }, @@ -296,10 +297,8 @@ { "icu", &lock_class_mtx_spin }, #ifdef SMP { "smp rendezvous", &lock_class_mtx_spin }, -#if defined(__i386__) && defined(APIC_IO) +#ifdef __i386__ { "tlb", &lock_class_mtx_spin }, -#endif -#ifdef __i386__ { "lazypmap", &lock_class_mtx_spin }, #endif #ifdef __sparc64__ Index: modules/acpi/Makefile =========================================================================== --- sys/modules/acpi/Makefile 2003/10/30 14:48:07 #21 +++ modules/acpi/Makefile 2003/10/30 14:48:07 @@ -66,6 +66,9 @@ # Machine-specific code such as sleep/wakeup SRCS+= acpi_machdep.c acpi_wakecode.h acpi_wakeup.c +.if ${MACHINE} == "i386" +SRCS+= madt.c +.endif CLEANFILES+= acpi_wakecode.h acpi_wakecode.o acpi_wakecode.bin acpi_wakecode.h: acpi_wakecode.S Index: notes =========================================================================== *** /dev/null Thu Oct 30 14:47:44 2003 --- notes Thu Oct 30 14:48:10 2003 *************** *** 0 **** --- 1,99 ---- + All files are under sys/i386 + + Files moved in this branch: + - include/apic.h -> include/apicreg.h + - isa/apic_vector.s -> i386/apic_vector.s + - isa/icu_vector.s -> isa/atpic_vector.s + - isa/intr_machdep.c -> isa/atpic.c (sort of) + - isa/intr_machdep.c -> isa/nmi.c + - isa/intr_machdep.h -> include/intr_machdep.h (sort of) + + New files: + - i386/local_apic.c + - i386/io_apic.c + - i386/mptable.c (from mp_machdep.c) + - i386/mptable_pci.c + - include/apicvar.h + - include/mptable.h (from mp_machdep.c) + + Files removed in this branch: + - conf/SMP + - i386/mpapic.c + - isa/intr_machdep.c + - isa/intr_machdep.h + - isa/icu_ipl.s + - isa/icu_vector.s + - isa/apic_ipl.s + - isa/apic_vector.s + - isa/ithread.c + - isa/vector.s + + Todo: + - DDB + + Add APIC ID to show pcpu + + Add 'show intrcnt' + + Add 'show idt' + - Add 'show ithread' + + Add 'show irqs' + - print out PIC driver name for each IRQ? + - Add 'show irq' + + Add nmi.c + + Set PCPU(apic_id) for UP case in lapic_init(). + + Use apic id of BSP (current CPU) when programming temporary physical + destinations into I/O APIC RDT entries. + + Fix IDT_SYSCALL mapping in APIC case + + Retest interrupts + - Somehow test IRQ's in this range? + + Test mixed mode + + works on SMP when single physical destination + + Fix bogus SMP probe when only 1 CPU + - Kill isa_irq_pending() (maybe, is this an MI function?) + + Add ACPI MADT APIC enumerator. + + solve ACPI module problem where madt wants APIC symbols :( + (this is gross) + - doesn't work when loaded as a module + - UMA needs to be fixed to not need mp_maxid so early + - other workaround is to fix mp_maxid at MAXCPU + - move cpu_mp_probe() to SI_SUB_CPU (perhaps shuffle SI of other + SMP sysinits as a result) + - update apic enumerators to probe CPUs later + - have apic enumerators hookup to the apic subsystem after SI_SUB_KLD + maybe SI_ORDER_LAST + - disable ACPI module for now + - Need to fix kernel profiling madness. Add an 'etrap' and perhaps + support for bintr2() and eintr2(). + + Implement mixed-mode and use it for IRQ0 unless no ExtINT pin or + NO_MIXED_MODE + + Add NO_MIXED_MODE option + - Add runtime decision for IRQ0 mixed mode? + + SMP bogons + + doesn't work on 750x + + pmap_lazyfix unhappy with old-style critical sections + - Chokes on scott's dual P3 when looking for ACPI RSDT. + - Perhaps change the IRQ resource manager such that it starts out empty + and a resource entry for each IRQ is added as each IRQ is registered + - Clean up includes in new files. + - Check #ifdef's in new files and look for #if 0 in new code + - Add handling of local APIC ERROR lvt. + - Use the local apic timer to drive hard/stat/profclock (maybe too hard) + - Do we need to be able to renumber I/O APICs? + - Check for overlap in table-specified APICs + - Ignore I/O APIC ID's if we have a local xAPIC (john cagle, Intel PDF) + - Renumber I/O APIC IDs. Ugh. Not sure if this will have to be done for + the general case or just for the mptable case. Update: may not have to, + it was a BIOS bug. May have to bump NAPICID to 256 though. + - Flesh out cpu topology map + - Suspend/resume support for I/O APICs + - check XXX and XXXTEST in new code + - reimplement lazy masking of interrupts for critical sections w/o bitmasks?? + - Enhance acpi_cpu(4) driver to grok SMP at all? + - Rip out CPU halting stuff perhaps and maybe disable MPTable HTT fixup? + - Support bus_config_intr()? + - nexus would have to call intr_config_source() + - intr_config_source() would call config_source() method of PIC driver + - AT PIC driver could just ignore + - APIC driver could reprogram I/O redirection entry + + Unrelated to this branch: + - Work on cleaning up nested includes in sys/* headers. According to Bruce, + the only nested sys/* headers should be _*.h and queue.h. Index: pc98/conf/GENERIC =========================================================================== --- sys/pc98/conf/GENERIC 2003/10/30 14:48:07 #58 +++ pc98/conf/GENERIC 2003/10/30 14:48:07 @@ -72,7 +72,7 @@ device isa device pci -#options COMPAT_OLDISA # Old ISA driver shims +options COMPAT_OLDISA # Old ISA driver shims # Floppy drives device fdc @@ -101,7 +101,7 @@ device aic # PC-9801-100 #options BS_TARG_SAFEMODE -#device bs 1 # WD33C93 SCSI card (55/92 like board) +device bs 1 # WD33C93 SCSI card (55/92 like board) device ct # host adapter using WD33C93[ABC] chip (C bus) device ncv # NCR 53C500 Index: setup.sh =========================================================================== *** /dev/null Thu Oct 30 14:47:44 2003 --- setup.sh Thu Oct 30 14:48:10 2003 *************** *** 0 **** --- 1,27 ---- + #!/bin/sh + # + # A simple script to create necessary symlinks for a kernel to build in this + # branch. + + # link oldname newname + # + # Where oldname is relative to the directory containing newname + link() + { + dir=`dirname $2` + old=$1 + new=`basename $2` + + (cd $dir ; ln -s $old $new ) + } + + # Make sure we are in the right place + if ! [ -r setup.sh ]; then + echo "This must be run from the top-level kernel source directory." + exit 1 + fi + + link apic.h i386/include/apicreg.h + link ../isa/apic_vector.s i386/i386/apic_vector.s + link icu_vector.s i386/isa/atpic_vector.s + link intr_machdep.c i386/isa/nmi.c