Index: linux-2.6.16-mm2/mm/migrate.c =================================================================== --- linux-2.6.16-mm2.orig/mm/migrate.c 2006-03-30 16:18:24.000000000 -0800 +++ linux-2.6.16-mm2/mm/migrate.c 2006-03-30 20:26:12.000000000 -0800 @@ -28,8 +28,6 @@ #include "internal.h" -#include "internal.h" - /* The maximum number of pages to take off the LRU for migration */ #define MIGRATE_CHUNK_SIZE 256 @@ -189,9 +187,13 @@ int migrate_page_remove_references(struc * indicates that the page is in use or truncate has removed * the page. */ - if (!mapping || page_mapcount(page) + nr_refs + - !!mapping != page_count(page)) + if (!page->mapping || + page_mapcount(page) + nr_refs + !!mapping != page_count(page)) { + printk(KERN_ERR "mapping or mapcount failure mapping=%p/%p refs=%d count=%d mapcount=%d\n", + page->mapping, mapping, nr_refs, page_count(page), page_mapcount(page)); + return -EAGAIN; + } /* * Establish swap ptes for anonymous pages or destroy pte @@ -216,21 +218,21 @@ int migrate_page_remove_references(struc /* A vma has VM_DONTMOVE set -> permanent failure */ return -EPERM; + if (!mapping) { + printk(KERN_ERR "Page %p references replaced.\n", page); + /* + * A anonymous page that is not on swap. We have removed all + * ptes and therefore are the only ones accessing the page. + */ + return 0; + } + /* * Give up if we were unable to remove all mappings. */ - if (page_mapcount(page)) + if (page_mapcount(page)) { + printk(KERN_ERR "After unmap: mapcount=%d\n", page_mapcount(page)); return -EAGAIN; - - if (!mapping) { - - /* A anonymous page that is not on swap. We must be the - * only one holding a reference. - */ - if (page_count(page) != nr_refs) - return -EAGAIN; - - return 0; } /* @@ -246,6 +248,7 @@ int migrate_page_remove_references(struc if (!page_mapping(page) || page_count(page) != nr_refs + 1 || *radix_pointer != page) { write_unlock_irq(&mapping->tree_lock); + printk(KERN_ERR "Final check failed.\n"); return -EAGAIN; } @@ -269,6 +272,7 @@ int migrate_page_remove_references(struc __put_page(page); write_unlock_irq(&mapping->tree_lock); + printk(KERN_ERR "Successful remove references for page %p with mapping.\n",mapping); return 0; } EXPORT_SYMBOL(migrate_page_remove_references); @@ -278,6 +282,8 @@ EXPORT_SYMBOL(migrate_page_remove_refere */ void migrate_page_copy(struct page *newpage, struct page *page) { + int mapcount; + copy_highpage(newpage, page); if (PageError(page)) @@ -298,12 +304,29 @@ void migrate_page_copy(struct page *newp set_page_dirty(newpage); } + newpage->index = page->index; + newpage->mapping = page->mapping; + + /* + * Anonymous pages preserve the mapcount but have removed the ptes. + * These need to be transferred to the newpage. + */ + mapcount = page_mapcount(page); + + set_page_count(newpage, page_count(newpage) + mapcount); + set_page_count(page, page_count(page) - mapcount); + + reset_page_mapcount(page); + reset_page_mapcount(newpage); + atomic_add(mapcount, &newpage->_mapcount); + ClearPageSwapCache(page); ClearPageActive(page); ClearPagePrivate(page); set_page_private(page, 0); page->mapping = NULL; + /* * If any waiters have accumulated on the new page then * wake them up. @@ -314,89 +337,6 @@ void migrate_page_copy(struct page *newp EXPORT_SYMBOL(migrate_page_copy); /* - * Restore a potential migration pte to a working pte entry for - * anonymous pages. - */ -void restore_pte(struct vm_area_struct *vma, unsigned long addr, - swp_entry_t entry, struct page *page) -{ - struct mm_struct *mm = vma->vm_mm; - pgd_t *pgd; - pud_t *pud; - pmd_t *pmd; - pte_t *ptep, pte; - spinlock_t *ptl; - - pgd = pgd_offset(mm, addr); - if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) - return; - - pud = pud_offset(pgd, addr); - if (pud_none(*pud) || unlikely(pud_bad(*pud))) - return; - - pmd = pmd_offset(pud, addr); - if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) - return; - - ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); - if (!ptep) - return; - - pte = *ptep; - - if (pte_present(pte) || pte_file(pte)) - goto out; - - if (pte_to_swp_entry(pte).val != entry.val) - goto out; - - set_pte_at(mm, addr, ptep, pte_mkold(mk_pte(page, vma->vm_page_prot))); - page_add_anon_rmap(page, vma, addr); - -out: - spin_unlock(ptl); -} - -/* - * Get rid of all migration entries and replace them by - * references to the indicated page. - * - * Must hold page lock on page and mmap_sem of one vma that contains - * the page. - */ -static void restore_ptes(struct page *page, struct page *newpage) -{ - struct anon_vma *anon_vma; - struct vm_area_struct *vma; - unsigned long mapping; - swp_entry_t entry; - - if (!PageSwapCache(page)) - return; - - mapping = (unsigned long)page->mapping; - - if (!mapping || (mapping & PAGE_MAPPING_ANON) == 0) - return; - - entry = swp_entry(SWP_TYPE_MIGRATION, page_to_pfn(page)); - - /* - * We hold the mmap_sem lock in some mm that this anon_vma belongs - * to. So no need to call page_lock_anon_vma. - */ - anon_vma = (struct anon_vma *) (mapping - PAGE_MAPPING_ANON); - spin_lock(&anon_vma->lock); - - list_for_each_entry(vma, &anon_vma->head, anon_vma_node) - restore_pte(vma, page_address_in_vma(page, vma), - entry, newpage); - - spin_unlock(&anon_vma->lock); -} - -/* * Common logic to directly migrate a single page suitable for * pages that do not use PagePrivate. * @@ -411,7 +351,7 @@ int migrate_page(struct page *newpage, s rc = migrate_page_remove_references(newpage, page, 1); if (rc) { - restore_ptes(page, page); + remove_migration_ptes(page, page); return rc; } @@ -425,9 +365,7 @@ int migrate_page(struct page *newpage, s * waiting on the page lock to use the new page via the page tables * before the new page is unlocked. */ - - /* Replace the special swap entries with real ptes */ - restore_ptes(page, newpage); + remove_migration_ptes(page, newpage); return 0; } EXPORT_SYMBOL(migrate_page); @@ -589,8 +527,10 @@ unlock_page: next: if (rc == -EAGAIN) { + printk(KERN_ERR "Migration Retry page %p Code=%d\n", page, rc); retry++; } else if (rc) { + printk(KERN_ERR "Migration Fail page %p Code=%d\n", page, rc); /* Permanent failure */ list_move(&page->lru, failed); nr_failed++; Index: linux-2.6.16-mm2/mm/rmap.c =================================================================== --- linux-2.6.16-mm2.orig/mm/rmap.c 2006-03-30 18:17:35.000000000 -0800 +++ linux-2.6.16-mm2/mm/rmap.c 2006-03-30 20:52:19.000000000 -0800 @@ -271,19 +271,20 @@ pte_t *page_check_address(struct page *p ptep = pte_offset_map(pmd, address); pte = *ptep; /* Make a quick check before getting the lock */ - if (!pte_present(pte)) { + if (pte_none(pte) || pte_file(pte)) { pte_unmap(ptep); return NULL; } ptl = pte_lockptr(mm, pmd); spin_lock(ptl); - if (pte_present(pte) && page_to_pfn(page) == pte_pfn(pte)) { - *ptlp = ptl; - return ptep; - } - /* Could still be a migration entry pointing to the page */ - if (!pte_present(pte) && !pte_file(pte)) { + if (pte_present(pte)) { + if (page_to_pfn(page) == pte_pfn(pte)) { + *ptlp = ptl; + return ptep; + } + } else { + /* Could still be a migration entry pointing to the page */ swp_entry_t entry = pte_to_swp_entry(pte); if (swp_type(entry) == SWP_TYPE_MIGRATION && @@ -297,6 +298,49 @@ pte_t *page_check_address(struct page *p } /* + * Restore a potential migration pte to a working pte entry for + * anonymous pages. + */ +static void remove_migration_pte(struct vm_area_struct *vma, unsigned long addr, + struct page *old, struct page *new) +{ + struct mm_struct *mm = vma->vm_mm; + pte_t *ptep; + spinlock_t *ptl; + + ptep = page_check_address(old, mm, addr, &ptl); + if (!ptep) + return; + + set_pte_at(mm, addr, ptep, pte_mkold(mk_pte(new, vma->vm_page_prot))); + spin_unlock(ptl); +} + +/* + * Get rid of all migration entries and replace them by + * references to the indicated page. + */ +void remove_migration_ptes(struct page *page, struct page *newpage) +{ + struct anon_vma *anon_vma; + struct vm_area_struct *vma; + + if (!PageAnon(newpage)) + return; + + printk(KERN_ERR "Removing migration ptes for page %p -> %p\n", page, newpage); + + anon_vma = page_lock_anon_vma(newpage); + BUG_ON(!anon_vma); + + list_for_each_entry(vma, &anon_vma->head, anon_vma_node) + remove_migration_pte(vma, page_address_in_vma(newpage, vma), + page, newpage); + + spin_unlock(&anon_vma->lock); +} + +/* * Subfunctions of page_referenced: page_referenced_one called * repeatedly from either page_referenced_anon or page_referenced_file. */ @@ -599,6 +643,11 @@ static int try_to_unmap_one(struct page swp_entry_t entry = { .val = page_private(page) }; if (!PageSwapCache(page) && migration) { + /* + * Store the pfn of the page in a special migration + * pte. do_swap_page will wait until the page is unlocked + * and then restart the fault handling. + */ entry = swp_entry(SWP_TYPE_MIGRATION, page_to_pfn(page)); set_pte_at(mm, address, pte, swp_entry_to_pte(entry)); goto out_unmap;