X86_64: Declare pda as per cpu data thereby moving it into the cpu area Declare the pda as a per cpu variable. This will have the effect of moving it into the cpu area managed by cpu alloc. After this was done the gs segment will point into the cpu area for each processor. The role of the boot_cpu_pda changes. It is only used in very early boot before setup_arch() completes. Signed-off-by: Christoph Lameter --- arch/x86/kernel/setup64.c | 32 +++++++--------------------- arch/x86/kernel/setup_64.c | 48 +++++++++++++++++++++++++++++++++++++++++++ arch/x86/kernel/smpboot_64.c | 16 -------------- include/asm-x86/percpu_64.h | 3 ++ 4 files changed, 59 insertions(+), 40 deletions(-) Index: linux-2.6/arch/x86/kernel/setup64.c =================================================================== --- linux-2.6.orig/arch/x86/kernel/setup64.c 2007-11-16 18:59:20.748451360 -0800 +++ linux-2.6/arch/x86/kernel/setup64.c 2007-11-16 19:04:11.104705217 -0800 @@ -30,7 +30,9 @@ cpumask_t cpu_initialized __cpuinitdata struct x8664_pda *_cpu_pda[NR_CPUS] __read_mostly; EXPORT_SYMBOL(_cpu_pda); -struct x8664_pda boot_cpu_pda[NR_CPUS] __cacheline_aligned; + +/* only used for very early boot (before setup_arch has finished) */ +struct x8664_pda __initdata boot_cpu_pda[NR_CPUS] ____cacheline_aligned_in_smp; struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table }; @@ -80,31 +82,13 @@ static int __init nonx32_setup(char *str __setup("noexec32=", nonx32_setup); /* - * Great future plan: - * Declare PDA itself and support (irqstack,tss,pgd) as per cpu data. - * Always point %gs to its beginning + * The setup of per cpu areas is done in setup_arch() now. + * + * But at that time the vm counters are not available. So we need + * to add the per cpu size here. */ void __init setup_per_cpu_areas(void) -{ - int i; - char *base; - -#ifdef CONFIG_HOTPLUG_CPU - prefill_possible_map(); -#endif - - /* Copy section for each CPU (we discard the original) */ - base = boot_cpu_alloc(PERCPU_ENOUGH_ROOM); - if (!base) - panic("Cannot allocate cpu data\n"); - - printk(KERN_INFO "PERCPU: Allocating %lu bytes of per cpu data\n", - PERCPU_ENOUGH_ROOM); - for_each_cpu_mask (i, cpu_possible_map) { - cpu_pda(i)->data_offset = CPU_PTR(base, i) - __per_cpu_start; - - memcpy(CPU_PTR(base, i), __per_cpu_start, __per_cpu_end - __per_cpu_start); - } +{ count_vm_events(CPU_BYTES, PERCPU_ENOUGH_ROOM); } Index: linux-2.6/arch/x86/kernel/smpboot_64.c =================================================================== --- linux-2.6.orig/arch/x86/kernel/smpboot_64.c 2007-11-16 17:30:18.734191679 -0800 +++ linux-2.6/arch/x86/kernel/smpboot_64.c 2007-11-16 18:59:21.956521033 -0800 @@ -556,22 +556,6 @@ static int __cpuinit do_boot_cpu(int cpu return -1; } - /* Allocate node local memory for AP pdas */ - if (cpu_pda(cpu) == &boot_cpu_pda[cpu]) { - struct x8664_pda *newpda, *pda; - int node = cpu_to_node(cpu); - pda = cpu_pda(cpu); - newpda = kmalloc_node(sizeof (struct x8664_pda), GFP_ATOMIC, - node); - if (newpda) { - memcpy(newpda, pda, sizeof (struct x8664_pda)); - cpu_pda(cpu) = newpda; - } else - printk(KERN_ERR - "Could not allocate node local PDA for CPU %d on node %d\n", - cpu, node); - } - alternatives_smp_switch(1); c_idle.idle = get_idle_for_cpu(cpu); Index: linux-2.6/arch/x86/kernel/setup_64.c =================================================================== --- linux-2.6.orig/arch/x86/kernel/setup_64.c 2007-11-16 17:30:18.742191619 -0800 +++ linux-2.6/arch/x86/kernel/setup_64.c 2007-11-16 18:59:21.960951662 -0800 @@ -254,6 +254,8 @@ static void discover_ebda(void) ebda_size = 64*1024; } +DEFINE_PER_CPU(struct x8664_pda, pda); + void __init setup_arch(char **cmdline_p) { printk(KERN_INFO "Command line: %s\n", boot_command_line); @@ -453,6 +455,52 @@ void __init setup_arch(char **cmdline_p) conswitchp = &dummy_con; #endif #endif +#ifdef CONFIG_HOTPLUG + prefill_possible_map(); +#endif + /* + * Create the per cpu areas and get the pda for cpu 0 operational. + * The boot pdas are no longer needed after this and all per cpu + * data will be accessible via gs segment override. + */ + { + unsigned i; + void *base; + + base = boot_cpu_alloc(ALIGN(__per_cpu_end - + __per_cpu_start, L1_CACHE_BYTES)); + + if (!cpu_pda(0)) + panic("Cannot allocate cpu data\n"); + + for_each_possible_cpu(i) { + char *this_area = CPU_PTR(base, i); + struct x8664_pda *this_pda; + + /* Copy per cpu data area for each processor */ + memcpy(this_area, __per_cpu_start, + __per_cpu_end - __per_cpu_start); + + /* Move the pda area into its proper location */ + this_pda = &per_cpu(pda, i); + memcpy(this_pda, cpu_pda(i), sizeof(struct x8664_pda)); + + /* + * Save the new pda address for code that indexes via + * cpu_pda. Code that runs both before and after the per cpu + * setup must index via cpu_pda(). + */ + cpu_pda(i) = this_pda; + + /* + * Setup the data offset field since we have the + * rest of the data handy + */ + this_pda->data_offset = this_area - __per_cpu_start; + } + } + + pda_init(0); } static int __cpuinit get_model_name(struct cpuinfo_x86 *c) Index: linux-2.6/include/asm-x86/percpu_64.h =================================================================== --- linux-2.6.orig/include/asm-x86/percpu_64.h 2007-11-16 17:30:18.754191568 -0800 +++ linux-2.6/include/asm-x86/percpu_64.h 2007-11-16 18:59:22.013226467 -0800 @@ -65,4 +65,7 @@ extern void setup_per_cpu_areas(void); #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var) +/* Avoid include hell */ +DECLARE_PER_CPU(struct x8664_pda, pda); + #endif /* _ASM_X8664_PERCPU_H_ */