497 lines
13 KiB
C
497 lines
13 KiB
C
/*-
|
|
* Copyright 2003-2011 Netlogic Microsystems (Netlogic). All rights
|
|
* reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are
|
|
* met:
|
|
*
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in
|
|
* the documentation and/or other materials provided with the
|
|
* distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY Netlogic Microsystems ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NETLOGIC OR CONTRIBUTORS BE
|
|
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
|
|
* THE POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
* NETLOGIC_BSD */
|
|
|
|
#include <sys/cdefs.h>
|
|
__FBSDID("$FreeBSD$");
|
|
#include <sys/types.h>
|
|
#include <sys/systm.h>
|
|
#include <sys/param.h>
|
|
#include <sys/lock.h>
|
|
#include <sys/mutex.h>
|
|
#include <sys/proc.h>
|
|
#include <sys/limits.h>
|
|
#include <sys/bus.h>
|
|
#include <sys/sbuf.h>
|
|
|
|
#include <sys/ktr.h>
|
|
#include <sys/kernel.h>
|
|
#include <sys/kthread.h>
|
|
#include <sys/proc.h>
|
|
#include <sys/resourcevar.h>
|
|
#include <sys/sched.h>
|
|
#include <sys/unistd.h>
|
|
#include <sys/sysctl.h>
|
|
#include <sys/malloc.h>
|
|
|
|
#include <machine/reg.h>
|
|
#include <machine/cpu.h>
|
|
#include <machine/hwfunc.h>
|
|
#include <machine/mips_opcode.h>
|
|
#include <machine/intr_machdep.h>
|
|
|
|
#include <mips/nlm/hal/mips-extns.h>
|
|
#include <mips/nlm/hal/haldefs.h>
|
|
#include <mips/nlm/hal/iomap.h>
|
|
#include <mips/nlm/hal/cop2.h>
|
|
#include <mips/nlm/hal/fmn.h>
|
|
#include <mips/nlm/hal/pic.h>
|
|
|
|
#include <mips/nlm/msgring.h>
|
|
#include <mips/nlm/interrupt.h>
|
|
#include <mips/nlm/xlp.h>
|
|
|
|
#define MSGRNG_NSTATIONS 1024
|
|
/*
|
|
* Keep track of our message ring handler threads, each core has a
|
|
* different message station. Ideally we will need to start a few
|
|
* message handling threads every core, and wake them up depending on
|
|
* load
|
|
*/
|
|
struct msgring_thread {
|
|
struct thread *thread; /* msgring handler threads */
|
|
int needed; /* thread needs to wake up */
|
|
};
|
|
static struct msgring_thread msgring_threads[XLP_MAX_CORES * XLP_MAX_THREADS];
|
|
static struct proc *msgring_proc; /* all threads are under a proc */
|
|
|
|
/*
|
|
* The device drivers can register a handler for the messages sent
|
|
* from a station (corresponding to the device).
|
|
*/
|
|
struct tx_stn_handler {
|
|
msgring_handler action;
|
|
void *arg;
|
|
};
|
|
static struct tx_stn_handler msgmap[MSGRNG_NSTATIONS];
|
|
static struct mtx msgmap_lock;
|
|
uint32_t xlp_msg_thread_mask;
|
|
static int xlp_msg_threads_per_core = XLP_MAX_THREADS;
|
|
|
|
static void create_msgring_thread(int hwtid);
|
|
static int msgring_process_fast_intr(void *arg);
|
|
|
|
/* Debug counters */
|
|
static int msgring_nintr[XLP_MAX_CORES * XLP_MAX_THREADS];
|
|
static int msgring_wakeup_sleep[XLP_MAX_CORES * XLP_MAX_THREADS];
|
|
static int msgring_wakeup_nosleep[XLP_MAX_CORES * XLP_MAX_THREADS];
|
|
static int fmn_msgcount[XLP_MAX_CORES * XLP_MAX_THREADS][4];
|
|
static int fmn_loops[XLP_MAX_CORES * XLP_MAX_THREADS];
|
|
|
|
/* Whether polled driver implementation */
|
|
static int polled = 0;
|
|
|
|
/* We do only i/o device credit setup here. CPU credit setup is now
|
|
* moved to xlp_msgring_cpu_init() so that the credits get setup
|
|
* only if the CPU exists. xlp_msgring_cpu_init() gets called from
|
|
* platform_init_ap; and this makes it easy for us to setup CMS
|
|
* credits for various types of XLP chips, with varying number of
|
|
* cpu's and cores.
|
|
*/
|
|
static void
|
|
xlp_cms_credit_setup(int credit)
|
|
{
|
|
uint64_t cmspcibase, cmsbase, pcibase;
|
|
uint32_t devoffset;
|
|
int dev, fn, maxqid;
|
|
int src, qid, i;
|
|
|
|
for (i = 0; i < XLP_MAX_NODES; i++) {
|
|
cmspcibase = nlm_get_cms_pcibase(i);
|
|
if (!nlm_dev_exists(XLP_IO_CMS_OFFSET(i)))
|
|
continue;
|
|
cmsbase = nlm_get_cms_regbase(i);
|
|
maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0);
|
|
for (dev = 0; dev < 8; dev++) {
|
|
for (fn = 0; fn < 8; fn++) {
|
|
devoffset = XLP_HDR_OFFSET(i, 0, dev, fn);
|
|
if (nlm_dev_exists(devoffset) == 0)
|
|
continue;
|
|
pcibase = nlm_pcicfg_base(devoffset);
|
|
src = nlm_qidstart(pcibase);
|
|
if (src == 0)
|
|
continue;
|
|
#if 0 /* Debug */
|
|
printf("Setup CMS credits for queues ");
|
|
printf("[%d to %d] from src %d\n", 0,
|
|
maxqid, src);
|
|
#endif
|
|
for (qid = 0; qid < maxqid; qid++)
|
|
nlm_cms_setup_credits(cmsbase, qid,
|
|
src, credit);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void
|
|
xlp_msgring_cpu_init(int node, int cpu, int credit)
|
|
{
|
|
uint64_t cmspcibase = nlm_get_cms_pcibase(node);
|
|
uint64_t cmsbase = nlm_get_cms_regbase(node);
|
|
int qid, maxqid, src;
|
|
|
|
maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0);
|
|
|
|
/* cpu credit setup is done only from thread-0 of each core */
|
|
if((cpu % 4) == 0) {
|
|
src = cpu << 2; /* each thread has 4 vc's */
|
|
for (qid = 0; qid < maxqid; qid++)
|
|
nlm_cms_setup_credits(cmsbase, qid, src, credit);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Drain out max_messages for the buckets set in the bucket mask.
|
|
* Use max_msgs = 0 to drain out all messages.
|
|
*/
|
|
int
|
|
xlp_handle_msg_vc(u_int vcmask, int max_msgs)
|
|
{
|
|
struct nlm_fmn_msg msg;
|
|
int srcid = 0, size = 0, code = 0;
|
|
struct tx_stn_handler *he;
|
|
uint32_t mflags, status;
|
|
int n_msgs = 0, vc, m, hwtid;
|
|
u_int msgmask;
|
|
|
|
hwtid = nlm_cpuid();
|
|
for (;;) {
|
|
/* check if VC empty */
|
|
mflags = nlm_save_flags_cop2();
|
|
status = nlm_read_c2_msgstatus1();
|
|
nlm_restore_flags(mflags);
|
|
|
|
msgmask = ((status >> 24) & 0xf) ^ 0xf;
|
|
msgmask &= vcmask;
|
|
if (msgmask == 0)
|
|
break;
|
|
m = 0;
|
|
for (vc = 0; vc < 4; vc++) {
|
|
if ((msgmask & (1 << vc)) == 0)
|
|
continue;
|
|
|
|
mflags = nlm_save_flags_cop2();
|
|
status = nlm_fmn_msgrcv(vc, &srcid, &size, &code,
|
|
&msg);
|
|
nlm_restore_flags(mflags);
|
|
if (status != 0) /* no msg or error */
|
|
continue;
|
|
if (srcid < 0 && srcid >= 1024) {
|
|
printf("[%s]: bad src id %d\n", __func__,
|
|
srcid);
|
|
continue;
|
|
}
|
|
he = &msgmap[srcid];
|
|
if(he->action != NULL)
|
|
(he->action)(vc, size, code, srcid, &msg,
|
|
he->arg);
|
|
#if 0
|
|
else
|
|
printf("[%s]: No Handler for msg from stn %d,"
|
|
" vc=%d, size=%d, msg0=%jx, droppinge\n",
|
|
__func__, srcid, vc, size,
|
|
(uintmax_t)msg.msg[0]);
|
|
#endif
|
|
fmn_msgcount[hwtid][vc] += 1;
|
|
m++; /* msgs handled in this iter */
|
|
}
|
|
if (m == 0)
|
|
break; /* nothing done in this iter */
|
|
n_msgs += m;
|
|
if (max_msgs > 0 && n_msgs >= max_msgs)
|
|
break;
|
|
}
|
|
|
|
return (n_msgs);
|
|
}
|
|
|
|
static void
|
|
xlp_discard_msg_vc(u_int vcmask)
|
|
{
|
|
struct nlm_fmn_msg msg;
|
|
int srcid = 0, size = 0, code = 0, vc;
|
|
uint32_t mflags, status;
|
|
|
|
for (vc = 0; vc < 4; vc++) {
|
|
for (;;) {
|
|
mflags = nlm_save_flags_cop2();
|
|
status = nlm_fmn_msgrcv(vc, &srcid,
|
|
&size, &code, &msg);
|
|
nlm_restore_flags(mflags);
|
|
|
|
/* break if there is no msg or error */
|
|
if (status != 0)
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
void
|
|
xlp_cms_enable_intr(int node, int cpu, int type, int watermark)
|
|
{
|
|
uint64_t cmsbase;
|
|
int i, qid;
|
|
|
|
cmsbase = nlm_get_cms_regbase(node);
|
|
|
|
for (i = 0; i < 4; i++) {
|
|
qid = (i + (cpu * 4)) & 0x7f;
|
|
nlm_cms_per_queue_level_intr(cmsbase, qid, type, watermark);
|
|
nlm_cms_per_queue_timer_intr(cmsbase, qid, 0x1, 0);
|
|
}
|
|
}
|
|
|
|
static int
|
|
msgring_process_fast_intr(void *arg)
|
|
{
|
|
struct msgring_thread *mthd;
|
|
struct thread *td;
|
|
int cpu;
|
|
|
|
cpu = nlm_cpuid();
|
|
mthd = &msgring_threads[cpu];
|
|
msgring_nintr[cpu]++;
|
|
td = mthd->thread;
|
|
|
|
/* clear pending interrupts */
|
|
nlm_write_c0_eirr(1ULL << IRQ_MSGRING);
|
|
|
|
/* wake up the target thread */
|
|
mthd->needed = 1;
|
|
thread_lock(td);
|
|
if (TD_AWAITING_INTR(td)) {
|
|
msgring_wakeup_sleep[cpu]++;
|
|
TD_CLR_IWAIT(td);
|
|
sched_add(td, SRQ_INTR);
|
|
} else
|
|
msgring_wakeup_nosleep[cpu]++;
|
|
|
|
thread_unlock(td);
|
|
|
|
return (FILTER_HANDLED);
|
|
}
|
|
|
|
static void
|
|
msgring_process(void * arg)
|
|
{
|
|
volatile struct msgring_thread *mthd;
|
|
struct thread *td;
|
|
uint32_t mflags, msgstatus1;
|
|
int hwtid, nmsgs;
|
|
|
|
hwtid = (intptr_t)arg;
|
|
mthd = &msgring_threads[hwtid];
|
|
td = mthd->thread;
|
|
KASSERT(curthread == td,
|
|
("%s:msg_ithread and proc linkage out of sync", __func__));
|
|
|
|
/* First bind this thread to the right CPU */
|
|
thread_lock(td);
|
|
sched_bind(td, xlp_hwtid_to_cpuid[hwtid]);
|
|
thread_unlock(td);
|
|
|
|
if (hwtid != nlm_cpuid())
|
|
printf("Misscheduled hwtid %d != cpuid %d\n", hwtid,
|
|
nlm_cpuid());
|
|
|
|
xlp_discard_msg_vc(0xf);
|
|
xlp_msgring_cpu_init(nlm_nodeid(), nlm_cpuid(), CMS_DEFAULT_CREDIT);
|
|
if (polled == 0) {
|
|
mflags = nlm_save_flags_cop2();
|
|
nlm_fmn_cpu_init(IRQ_MSGRING, 0, 0, 0, 0, 0);
|
|
nlm_restore_flags(mflags);
|
|
xlp_cms_enable_intr(nlm_nodeid(), nlm_cpuid(), 0x2, 0);
|
|
/* clear pending interrupts.
|
|
* they will get re-raised if still valid */
|
|
nlm_write_c0_eirr(1ULL << IRQ_MSGRING);
|
|
}
|
|
|
|
/* start processing messages */
|
|
for (;;) {
|
|
atomic_store_rel_int(&mthd->needed, 0);
|
|
nmsgs = xlp_handle_msg_vc(0xf, 0);
|
|
|
|
/* sleep */
|
|
if (polled == 0) {
|
|
/* clear VC-pend bits */
|
|
mflags = nlm_save_flags_cop2();
|
|
msgstatus1 = nlm_read_c2_msgstatus1();
|
|
msgstatus1 |= (0xf << 16);
|
|
nlm_write_c2_msgstatus1(msgstatus1);
|
|
nlm_restore_flags(mflags);
|
|
|
|
thread_lock(td);
|
|
if (mthd->needed) {
|
|
thread_unlock(td);
|
|
continue;
|
|
}
|
|
sched_class(td, PRI_ITHD);
|
|
TD_SET_IWAIT(td);
|
|
mi_switch(SW_VOL, NULL);
|
|
thread_unlock(td);
|
|
} else
|
|
pause("wmsg", 1);
|
|
|
|
fmn_loops[hwtid]++;
|
|
}
|
|
}
|
|
|
|
static void
|
|
create_msgring_thread(int hwtid)
|
|
{
|
|
struct msgring_thread *mthd;
|
|
struct thread *td;
|
|
int error;
|
|
|
|
mthd = &msgring_threads[hwtid];
|
|
error = kproc_kthread_add(msgring_process, (void *)(uintptr_t)hwtid,
|
|
&msgring_proc, &td, RFSTOPPED, 2, "msgrngproc",
|
|
"msgthr%d", hwtid);
|
|
if (error)
|
|
panic("kproc_kthread_add() failed with %d", error);
|
|
mthd->thread = td;
|
|
|
|
thread_lock(td);
|
|
sched_class(td, PRI_ITHD);
|
|
sched_add(td, SRQ_INTR);
|
|
thread_unlock(td);
|
|
}
|
|
|
|
int
|
|
register_msgring_handler(int startb, int endb, msgring_handler action,
|
|
void *arg)
|
|
{
|
|
int i;
|
|
|
|
if (bootverbose)
|
|
printf("Register handler %d-%d %p(%p)\n",
|
|
startb, endb, action, arg);
|
|
KASSERT(startb >= 0 && startb <= endb && endb < MSGRNG_NSTATIONS,
|
|
("Invalid value for bucket range %d,%d", startb, endb));
|
|
|
|
mtx_lock_spin(&msgmap_lock);
|
|
for (i = startb; i <= endb; i++) {
|
|
KASSERT(msgmap[i].action == NULL,
|
|
("Bucket %d already used [action %p]", i, msgmap[i].action));
|
|
msgmap[i].action = action;
|
|
msgmap[i].arg = arg;
|
|
}
|
|
mtx_unlock_spin(&msgmap_lock);
|
|
return (0);
|
|
}
|
|
|
|
/*
|
|
* Initialize the messaging subsystem.
|
|
*
|
|
* Message Stations are shared among all threads in a cpu core, this
|
|
* has to be called once from every core which is online.
|
|
*/
|
|
static void
|
|
xlp_msgring_config(void *arg)
|
|
{
|
|
void *cookie;
|
|
unsigned int thrmask, mask;
|
|
int i;
|
|
|
|
/* used polled handler for Ax silion */
|
|
if (nlm_is_xlp8xx_ax())
|
|
polled = 1;
|
|
|
|
/* Don't poll on all threads, if polled */
|
|
if (polled)
|
|
xlp_msg_threads_per_core -= 1;
|
|
|
|
mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN);
|
|
if (xlp_threads_per_core < xlp_msg_threads_per_core)
|
|
xlp_msg_threads_per_core = xlp_threads_per_core;
|
|
thrmask = ((1 << xlp_msg_threads_per_core) - 1);
|
|
mask = 0;
|
|
for (i = 0; i < XLP_MAX_CORES; i++) {
|
|
mask <<= XLP_MAX_THREADS;
|
|
mask |= thrmask;
|
|
}
|
|
xlp_msg_thread_mask = xlp_hw_thread_mask & mask;
|
|
#if 0
|
|
printf("CMS Message handler thread mask %#jx\n",
|
|
(uintmax_t)xlp_msg_thread_mask);
|
|
#endif
|
|
xlp_cms_credit_setup(CMS_DEFAULT_CREDIT);
|
|
create_msgring_thread(0);
|
|
cpu_establish_hardintr("msgring", msgring_process_fast_intr, NULL,
|
|
NULL, IRQ_MSGRING, INTR_TYPE_NET, &cookie);
|
|
}
|
|
|
|
/*
|
|
* Start message ring processing threads on other CPUs, after SMP start
|
|
*/
|
|
static void
|
|
start_msgring_threads(void *arg)
|
|
{
|
|
int hwt;
|
|
|
|
for (hwt = 1; hwt < XLP_MAX_CORES * XLP_MAX_THREADS; hwt++) {
|
|
if ((xlp_msg_thread_mask & (1 << hwt)) == 0)
|
|
continue;
|
|
create_msgring_thread(hwt);
|
|
}
|
|
}
|
|
|
|
SYSINIT(xlp_msgring_config, SI_SUB_DRIVERS, SI_ORDER_FIRST,
|
|
xlp_msgring_config, NULL);
|
|
SYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE,
|
|
start_msgring_threads, NULL);
|
|
|
|
/*
|
|
* DEBUG support, XXX: static buffer, not locked
|
|
*/
|
|
static int
|
|
sys_print_debug(SYSCTL_HANDLER_ARGS)
|
|
{
|
|
struct sbuf sb;
|
|
int error, i;
|
|
|
|
sbuf_new_for_sysctl(&sb, NULL, 64, req);
|
|
sbuf_printf(&sb,
|
|
"\nID vc0 vc1 vc2 vc3 loops\n");
|
|
for (i = 0; i < 32; i++) {
|
|
if ((xlp_hw_thread_mask & (1 << i)) == 0)
|
|
continue;
|
|
sbuf_printf(&sb, "%2d: %8d %8d %8d %8d %8d\n", i,
|
|
fmn_msgcount[i][0], fmn_msgcount[i][1],
|
|
fmn_msgcount[i][2], fmn_msgcount[i][3],
|
|
fmn_loops[i]);
|
|
}
|
|
error = sbuf_finish(&sb);
|
|
sbuf_delete(&sb);
|
|
return (error);
|
|
}
|
|
|
|
SYSCTL_PROC(_debug, OID_AUTO, msgring, CTLTYPE_STRING | CTLFLAG_RD, 0, 0,
|
|
sys_print_debug, "A", "msgring debug info");
|