patch-2.4.19 linux-2.4.19/arch/mips/kernel/smp.c

• Lines: 574
• Date: Fri Aug 2 17:39:43 2002
• Orig file: linux-2.4.18/arch/mips/kernel/smp.c
• Orig date: Fri Dec 21 09:41:53 2001

diff -urN linux-2.4.18/arch/mips/kernel/smp.c linux-2.4.19/arch/mips/kernel/smp.c
@@ -1,11 +1,4 @@
 /*
- *
- *  arch/mips/kernel/smp.c
- *
- *  Copyright (C) 2000 Sibyte
- * 
- *  Written by Justin Carlson (carlson@sibyte.com)
- *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version 2
@@ -20,20 +13,24 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  *
+ * Copyright (C) 2000, 2001 Kanoj Sarcar
+ * Copyright (C) 2000, 2001 Ralf Baechle
+ * Copyright (C) 2000, 2001 Silicon Graphics, Inc.
+ * Copyright (C) 2000, 2001 Broadcom Corporation
  */ 
-
-
-#include <linux/config.h>
+#include <linux/cache.h>
 #include <linux/init.h>
 #include <linux/spinlock.h>
 #include <linux/threads.h>
 #include <linux/time.h>
+#include <linux/module.h>
 #include <linux/timex.h>
 #include <linux/sched.h>
 #include <linux/interrupt.h>
-#include <linux/cache.h>
+#include <linux/mm.h>
 
 #include <asm/atomic.h>
+#include <asm/cpu.h>
 #include <asm/processor.h>
 #include <asm/system.h>
 #include <asm/hardirq.h>
@@ -42,38 +39,29 @@
 #include <asm/delay.h>
 #include <asm/smp.h>
 
-/*
- * This was written with the BRCM12500 MP SOC in mind, but tries to
- * be generic.  It's modelled on the mips64 smp.c code, which is
- * derived from Sparc, I'm guessing, which is derived from...
- * 
- * It's probably horribly designed for very large ccNUMA systems
- * as it doesn't take any node clustering into account.  
-*/
-
-
 /* Ze Big Kernel Lock! */
 spinlock_t kernel_flag __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
-int smp_threads_ready;  /* Not used */
-int smp_num_cpus;    
-int global_irq_holder = NO_PROC_ID;
-spinlock_t global_irq_lock = SPIN_LOCK_UNLOCKED;
-struct mips_cpuinfo cpu_data[NR_CPUS];
-
-struct smp_fn_call_struct smp_fn_call = 
-{ SPIN_LOCK_UNLOCKED, ATOMIC_INIT(0), NULL, NULL};
+int smp_threads_ready;
+atomic_t smp_commenced = ATOMIC_INIT(0);
+int smp_num_cpus = 1;			/* Number that came online.  */
+cpumask_t cpu_online_map;		/* Bitmask of currently online CPUs */
+int __cpu_number_map[NR_CPUS];
+int __cpu_logical_map[NR_CPUS];
+struct cpuinfo_mips cpu_data[NR_CPUS];
+void (*volatile smp_cpu0_finalize)(void);
 
-static atomic_t cpus_booted = ATOMIC_INIT(0);
+// static atomic_t cpus_booted = ATOMIC_INIT(0);
+atomic_t cpus_booted = ATOMIC_INIT(0);
 
 
 /* These are defined by the board-specific code. */
 
-/* Cause the function described by smp_fn_call 
-   to be executed on the passed cpu.  When the function
-   has finished, increment the finished field of
-   smp_fn_call. */
-
-void core_call_function(int cpu);
+/*
+ * Cause the function described by call_data to be executed on the passed
+ * cpu.  When the function has finished, increment the finished field of
+ * call_data.
+ */
+void core_send_ipi(int cpu, unsigned int action);
 
 /*
  * Clear all undefined state in the cpu, set up sp and gp to the passed
@@ -85,108 +73,54 @@
  *  After we've done initial boot, this function is called to allow the
  *  board code to clean up state, if needed 
  */
-
 void prom_init_secondary(void);
 
-
-void cpu_idle(void);
-
-/* Do whatever setup needs to be done for SMP at the board level.  Return
-   the number of cpus in the system, including this one */
+/*
+ * Do whatever setup needs to be done for SMP at the board level.  Return
+ * the number of cpus in the system, including this one
+ */
 int prom_setup_smp(void);
 
-int start_secondary(void *unused)
+/*
+ * Hook for doing final board-specific setup after the generic smp setup
+ * is done
+ */
+asmlinkage void start_secondary(void)
 {
+	unsigned int cpu = smp_processor_id();
+
 	prom_init_secondary();
-	write_32bit_cp0_register(CP0_CONTEXT, smp_processor_id()<<23);
-	current_pgd[smp_processor_id()] = init_mm.pgd;
+	per_cpu_trap_init();
+
+	/*
+	 * XXX parity protection should be folded in here when it's converted
+	 * to an option instead of something based on .cputype
+	 */
+
+	pgd_current[cpu] = init_mm.pgd;
+	cpu_data[cpu].udelay_val = loops_per_jiffy;
+	prom_smp_finish();
 	printk("Slave cpu booted successfully\n");
+	CPUMASK_SETB(cpu_online_map, cpu);
 	atomic_inc(&cpus_booted);
+	while (!atomic_read(&smp_commenced));
 	cpu_idle();
-	return 0;
-}
-
-void __init smp_boot_cpus(void)
-{
-	int i;
-
-	smp_num_cpus = prom_setup_smp();
-	init_new_context(current, &init_mm);
-	current->processor = 0;
-	atomic_set(&cpus_booted, 1);  /* Master CPU is already booted... */
-	init_idle();
-	for (i = 1; i < smp_num_cpus; i++) {
-		struct task_struct *p;
-		struct pt_regs regs;
-		printk("Starting CPU %d... ", i);
-
-		/* Spawn a new process normally.  Grab a pointer to
-		   its task struct so we can mess with it */
-		do_fork(CLONE_VM|CLONE_PID, 0, &regs, 0);
-		p = init_task.prev_task;
-
-		/* Schedule the first task manually */
-		p->processor = i;
-		p->cpus_runnable = 1 << i; /* we schedule the first task manually */
-
-		/* Attach to the address space of init_task. */
-		atomic_inc(&init_mm.mm_count);
-		p->active_mm = &init_mm;
-		init_tasks[i] = p;
-
-		del_from_runqueue(p);
-		unhash_process(p);
-
-		prom_boot_secondary(i,
-				    (unsigned long)p + KERNEL_STACK_SIZE - 32,
-				    (unsigned long)p);
-
-#if 0
-
-		/* This is copied from the ip-27 code in the mips64 tree */
-
-		struct task_struct *p;
-
-		/*
-		 * The following code is purely to make sure
-		 * Linux can schedule processes on this slave.
-		 */
-		kernel_thread(0, NULL, CLONE_PID);
-		p = init_task.prev_task;
-		sprintf(p->comm, "%s%d", "Idle", i);
-		init_tasks[i] = p;
-		p->processor = i;
-		p->cpus_runnable = 1 << i; /* we schedule the first task manually *
-		del_from_runqueue(p);
-		unhash_process(p);
-		/* Attach to the address space of init_task. */
-		atomic_inc(&init_mm.mm_count);
-		p->active_mm = &init_mm;
-		prom_boot_secondary(i, 
-				    (unsigned long)p + KERNEL_STACK_SIZE - 32,
-				    (unsigned long)p);
-#endif
-	}
-
-	/* Wait for everyone to come up */
-	while (atomic_read(&cpus_booted) != smp_num_cpus);
 }
 
 void __init smp_commence(void)
 {
-	/* Not sure what to do here yet */
+	wmb();
+	atomic_set(&smp_commenced, 1);
 }
 
-static void reschedule_this_cpu(void *dummy)
+void smp_send_reschedule(int cpu)
 {
-	current->need_resched = 1;
+	core_send_ipi(cpu, SMP_RESCHEDULE_YOURSELF);
 }
 
-void FASTCALL(smp_send_reschedule(int cpu))
-{
-	smp_call_function(reschedule_this_cpu, NULL, 0, 0);
-}
+static spinlock_t call_lock = SPIN_LOCK_UNLOCKED;
 
+struct call_data_struct *call_data;
 
 /*
  * The caller of this wants the passed function to run on every cpu.  If wait
@@ -196,43 +130,76 @@
 int smp_call_function (void (*func) (void *info), void *info, int retry, 
 								int wait)
 {
-	int cpus = smp_num_cpus - 1;
-	int i;
+	struct call_data_struct data;
+	int i, cpus = smp_num_cpus - 1;
+	int cpu = smp_processor_id();
 
-	if (smp_num_cpus < 2) {
+	if (!cpus)
 		return 0;
-	}
 
-	spin_lock_bh(&smp_fn_call.lock);
+	data.func = func;
+	data.info = info;
+	atomic_set(&data.started, 0);
+	data.wait = wait;
+	if (wait)
+		atomic_set(&data.finished, 0);
+
+	spin_lock(&call_lock);
+	call_data = &data;
+
+	/* Send a message to all other CPUs and wait for them to respond */
+	for (i = 0; i < smp_num_cpus; i++)
+		if (i != cpu)
+			core_send_ipi(i, SMP_CALL_FUNCTION);
+
+	/* Wait for response */
+	/* FIXME: lock-up detection, backtrace on lock-up */
+	while (atomic_read(&data.started) != cpus)
+		barrier();
+
+	if (wait)
+		while (atomic_read(&data.finished) != cpus)
+			barrier();
+	spin_unlock(&call_lock);
 
-	atomic_set(&smp_fn_call.finished, 0);
-	smp_fn_call.fn = func;
-	smp_fn_call.data = info;
-
-	for (i = 0; i < smp_num_cpus; i++) {
-		if (i != smp_processor_id()) {
-			/* Call the board specific routine */
-			core_call_function(i);
-		}
-	}
-
-	if (wait) {
-		while(atomic_read(&smp_fn_call.finished) != cpus) {}
-	}
-
-	spin_unlock_bh(&smp_fn_call.lock);
 	return 0;
 }
 
-void synchronize_irq(void)
+void smp_call_function_interrupt(void)
 {
-	panic("synchronize_irq");
+	void (*func) (void *info) = call_data->func;
+	void *info = call_data->info;
+	int wait = call_data->wait;
+	int cpu = smp_processor_id();
+
+	irq_enter(cpu, 0);	/* XXX choose an irq number? */
+	/*
+	 * Notify initiating CPU that I've grabbed the data
+	 * and am about to execute the function
+	 */
+	mb();
+	atomic_inc(&call_data->started);
+
+	/*
+	 * At this point the info structure may be out of scope unless wait==1
+	 */
+	(*func)(info);
+	if (wait) {
+		mb();
+		atomic_inc(&call_data->finished);
+	}
+	irq_exit(cpu, 0);	/* XXX choose an irq number? */
 }
 
 static void stop_this_cpu(void *dummy)
 {
-	printk("Cpu stopping\n");
-	for (;;);
+	int cpu = smp_processor_id();
+	if (cpu)
+		for (;;);		/* XXX Use halt like i386 */
+
+	/* XXXKW this isn't quite there yet */
+	while (!smp_cpu0_finalize) ;
+	smp_cpu0_finalize();
 }
 
 void smp_send_stop(void)
@@ -247,157 +214,110 @@
 	return 0;
 }
 
+static void flush_tlb_all_ipi(void *info)
+{
+	local_flush_tlb_all();
+}
 
-/*
- * Most of this code is take from the mips64 tree (ip27-irq.c).  It's virtually
- * identical to the i386 implentation in arh/i386/irq.c, with translations for
- * the interrupt enable bit
- */
-
-#define MAXCOUNT 		100000000
-#define SYNC_OTHER_CORES(x)	udelay(x+1)
+void flush_tlb_all(void)
+{
+	smp_call_function(flush_tlb_all_ipi, 0, 1, 1);
+	local_flush_tlb_all();
+}
 
-static inline void wait_on_irq(int cpu)
+static void flush_tlb_mm_ipi(void *mm)
 {
-	int count = MAXCOUNT;
+	local_flush_tlb_mm((struct mm_struct *)mm);
+}
 
-	for (;;) {
+/*
+ * The following tlb flush calls are invoked when old translations are 
+ * being torn down, or pte attributes are changing. For single threaded
+ * address spaces, a new context is obtained on the current cpu, and tlb
+ * context on other cpus are invalidated to force a new context allocation
+ * at switch_mm time, should the mm ever be used on other cpus. For 
+ * multithreaded address spaces, intercpu interrupts have to be sent.
+ * Another case where intercpu interrupts are required is when the target
+ * mm might be active on another cpu (eg debuggers doing the flushes on
+ * behalf of debugees, kswapd stealing pages from another process etc).
+ * Kanoj 07/00.
+ */
 
-		/*
-		 * Wait until all interrupts are gone. Wait
-		 * for bottom half handlers unless we're
-		 * already executing in one..
-		 */
-		if (!irqs_running())
-			if (local_bh_count(cpu) || !spin_is_locked(&global_bh_lock))
-				break;
-
-		/* Duh, we have to loop. Release the lock to avoid deadlocks */
-		spin_unlock(&global_irq_lock);
-
-		for (;;) {
-			if (!--count) {
-				printk("Count spun out.  Huh?\n");
-				count = ~0;
-			}
-			__sti();
-			SYNC_OTHER_CORES(cpu);
-			__cli();
-			if (irqs_running())
-				continue;
-			if (spin_is_locked(&global_irq_lock))
-				continue;
-			if (!local_bh_count(cpu) && spin_is_locked(&global_bh_lock))
-				continue;
-			if (spin_trylock(&global_irq_lock))
-				break;
-		}
+void flush_tlb_mm(struct mm_struct *mm)
+{
+	if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
+		smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1, 1);
+	} else {
+		int i;
+		for (i = 0; i < smp_num_cpus; i++)
+			if (smp_processor_id() != i)
+				CPU_CONTEXT(i, mm) = 0;
 	}
+	local_flush_tlb_mm(mm);
 }
 
+struct flush_tlb_data {
+	struct mm_struct *mm;
+	struct vm_area_struct *vma;
+	unsigned long addr1;
+	unsigned long addr2;
+};
 
-static inline void get_irqlock(int cpu)
+static void flush_tlb_range_ipi(void *info)
 {
-	if (!spin_trylock(&global_irq_lock)) {
-		/* do we already hold the lock? */
-		if ((unsigned char) cpu == global_irq_holder)
-			return;
-		/* Uhhuh.. Somebody else got it. Wait.. */
-		spin_lock(&global_irq_lock);
-	}
-	/*
-	 * We also to make sure that nobody else is running
-	 * in an interrupt context.
-	 */
-	wait_on_irq(cpu);
+	struct flush_tlb_data *fd = (struct flush_tlb_data *)info;
 
-	/*
-	 * Ok, finally..
-	 */
-	global_irq_holder = cpu;
+	local_flush_tlb_range(fd->mm, fd->addr1, fd->addr2);
 }
 
-
-/*
- * A global "cli()" while in an interrupt context
- * turns into just a local cli(). Interrupts
- * should use spinlocks for the (very unlikely)
- * case that they ever want to protect against
- * each other.
- *
- * If we already have local interrupts disabled,
- * this will not turn a local disable into a
- * global one (problems with spinlocks: this makes
- * save_flags+cli+sti usable inside a spinlock).
- */
-void __global_cli(void)
+void flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long end)
 {
-	unsigned int flags;
+	if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
+		struct flush_tlb_data fd;
 
-	__save_flags(flags);
-	if (flags & ST0_IE) {
-		int cpu = smp_processor_id();
-		__cli();
-		if (!local_irq_count(cpu))
-			get_irqlock(cpu);
+		fd.mm = mm;
+		fd.addr1 = start;
+		fd.addr2 = end;
+		smp_call_function(flush_tlb_range_ipi, (void *)&fd, 1, 1);
+	} else {
+		int i;
+		for (i = 0; i < smp_num_cpus; i++)
+			if (smp_processor_id() != i)
+				CPU_CONTEXT(i, mm) = 0;
 	}
+	local_flush_tlb_range(mm, start, end);
 }
 
-void __global_sti(void)
+static void flush_tlb_page_ipi(void *info)
 {
-	int cpu = smp_processor_id();
+	struct flush_tlb_data *fd = (struct flush_tlb_data *)info;
 
-	if (!local_irq_count(cpu))
-		release_irqlock(cpu);
-	__sti();
+	local_flush_tlb_page(fd->vma, fd->addr1);
 }
 
-/*
- * SMP flags value to restore to:
- * 0 - global cli
- * 1 - global sti
- * 2 - local cli
- * 3 - local sti
- */
-unsigned long __global_save_flags(void)
+void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 {
-	int retval;
-	int local_enabled;
-	unsigned long flags;
-	int cpu = smp_processor_id();
+	if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
+		struct flush_tlb_data fd;
 
-	__save_flags(flags);
-	local_enabled = (flags & ST0_IE);
-	/* default to local */
-	retval = 2 + local_enabled;
-
-	/* check for global flags if we're not in an interrupt */
-	if (!local_irq_count(cpu)) {
-		if (local_enabled)
-			retval = 1;
-		if (global_irq_holder == cpu)
-			retval = 0;
+		fd.vma = vma;
+		fd.addr1 = page;
+		smp_call_function(flush_tlb_page_ipi, (void *)&fd, 1, 1);
+	} else {
+		int i;
+		for (i = 0; i < smp_num_cpus; i++)
+			if (smp_processor_id() != i)
+				CPU_CONTEXT(i, vma->vm_mm) = 0;
 	}
-
-	return retval;
+	local_flush_tlb_page(vma, page);
 }
 
-void __global_restore_flags(unsigned long flags)
-{
-	switch (flags) {
-		case 0:
-			__global_cli();
-			break;
-		case 1:
-			__global_sti();
-			break;
-		case 2:
-			__cli();
-			break;
-		case 3:
-			__sti();
-			break;
-		default:
-			printk("global_restore_flags: %08lx\n", flags);
-	}
-}
+EXPORT_SYMBOL(smp_num_cpus);
+EXPORT_SYMBOL(flush_tlb_page);
+EXPORT_SYMBOL(cpu_data);
+EXPORT_SYMBOL(synchronize_irq);
+EXPORT_SYMBOL(kernel_flag);
+EXPORT_SYMBOL(__global_sti);
+EXPORT_SYMBOL(__global_cli);
+EXPORT_SYMBOL(__global_save_flags);
+EXPORT_SYMBOL(__global_restore_flags);