patch-2.4.5 linux/mm/vmscan.c

• Lines: 324
• Date: Fri May 25 17:00:18 2001
• Orig file: v2.4.4/linux/mm/vmscan.c
• Orig date: Thu Apr 12 12:10:25 2001

diff -u --recursive --new-file v2.4.4/linux/mm/vmscan.c linux/mm/vmscan.c
@@ -24,6 +24,8 @@
 
 #include <asm/pgalloc.h>
 
+#define MAX(a,b) ((a) > (b) ? (a) : (b))
+
 /*
  * The swap-out function returns 1 if it successfully
  * scanned all the pages it was asked to (`count').
@@ -228,6 +230,8 @@
 	unsigned long address;
 	struct vm_area_struct* vma;
 
+	if (!count)
+		return 1;
 	/*
 	 * Go through process' page directory.
 	 */
@@ -271,7 +275,12 @@
 static inline int swap_amount(struct mm_struct *mm)
 {
 	int nr = mm->rss >> SWAP_SHIFT;
-	return nr < SWAP_MIN ? SWAP_MIN : nr;
+	if (nr < SWAP_MIN) {
+		nr = SWAP_MIN;
+		if (nr > mm->rss)
+			nr = mm->rss;
+	}
+	return nr;
 }
 
 static int swap_out(unsigned int priority, int gfp_mask)
@@ -285,7 +294,7 @@
 		retval = swap_out_mm(mm, swap_amount(mm));
 
 	/* Then, look at the other mm's */
-	counter = mmlist_nr >> priority;
+	counter = (mmlist_nr << SWAP_SHIFT) >> priority;
 	do {
 		struct list_head *p;
 
@@ -350,7 +359,7 @@
 		}
 
 		/* Page is or was in use?  Move it to the active list. */
-		if (PageTestandClearReferenced(page) || page->age > 0 ||
+		if (PageReferenced(page) || page->age > 0 ||
 				(!page->buffers && page_count(page) > 1)) {
 			del_page_from_inactive_clean_list(page);
 			add_page_to_active_list(page);
@@ -453,7 +462,7 @@
 		}
 
 		/* Page is or was in use?  Move it to the active list. */
-		if (PageTestandClearReferenced(page) || page->age > 0 ||
+		if (PageReferenced(page) || page->age > 0 ||
 				(!page->buffers && page_count(page) > 1) ||
 				page_ramdisk(page)) {
 			del_page_from_inactive_dirty_list(page);
@@ -631,21 +640,42 @@
 /**
  * refill_inactive_scan - scan the active list and find pages to deactivate
  * @priority: the priority at which to scan
- * @oneshot: exit after deactivating one page
+ * @target: number of pages to deactivate, zero for background aging
  *
  * This function will scan a portion of the active list to find
  * unused pages, those pages will then be moved to the inactive list.
  */
-int refill_inactive_scan(unsigned int priority, int oneshot)
+int refill_inactive_scan(unsigned int priority, int target)
 {
 	struct list_head * page_lru;
 	struct page * page;
-	int maxscan, page_active = 0;
-	int ret = 0;
+	int maxscan = nr_active_pages >> priority;
+	int page_active = 0;
+	int nr_deactivated = 0;
+
+	/*
+	 * When we are background aging, we try to increase the page aging
+	 * information in the system. When we have too many inactive pages
+	 * we don't do background aging since having all pages on the
+	 * inactive list decreases aging information.
+	 *
+	 * Since not all active pages have to be on the active list, we round
+	 * nr_active_pages up to num_physpages/2, if needed.
+	 */
+	if (!target) {
+		int inactive = nr_free_pages() + nr_inactive_clean_pages() +
+						nr_inactive_dirty_pages;
+		int active = MAX(nr_active_pages, num_physpages / 2);
+		if (active > 10 * inactive)
+			maxscan = nr_active_pages >> 4;
+		else if (active > 3 * inactive)
+			maxscan = nr_active_pages >> 8;
+		else
+			return 0;
+	}
 
 	/* Take the lock while messing with the list... */
 	spin_lock(&pagemap_lru_lock);
-	maxscan = nr_active_pages >> priority;
 	while (maxscan-- > 0 && (page_lru = active_list.prev) != &active_list) {
 		page = list_entry(page_lru, struct page, lru);
 
@@ -683,21 +713,21 @@
 		}
 		/*
 		 * If the page is still on the active list, move it
-		 * to the other end of the list. Otherwise it was
-		 * deactivated by age_page_down and we exit successfully.
+		 * to the other end of the list. Otherwise we exit if
+		 * we have done enough work.
 		 */
 		if (page_active || PageActive(page)) {
 			list_del(page_lru);
 			list_add(page_lru, &active_list);
 		} else {
-			ret = 1;
-			if (oneshot)
+			nr_deactivated++;
+			if (target && nr_deactivated >= target)
 				break;
 		}
 	}
 	spin_unlock(&pagemap_lru_lock);
 
-	return ret;
+	return nr_deactivated;
 }
 
 /*
@@ -709,7 +739,7 @@
 	pg_data_t *pgdat = pgdat_list;
 	int sum = 0;
 	int freeable = nr_free_pages() + nr_inactive_clean_pages();
-	int freetarget = freepages.high + inactive_target / 3;
+	int freetarget = freepages.high;
 
 	/* Are we low on free pages globally? */
 	if (freeable < freetarget)
@@ -721,9 +751,8 @@
 		for(i = 0; i < MAX_NR_ZONES; i++) {
 			zone_t *zone = pgdat->node_zones+ i;
 			if (zone->size && (zone->inactive_clean_pages +
-					zone->free_pages < zone->pages_min+1)) {
-				/* + 1 to have overlap with alloc_pages() !! */
-				sum += zone->pages_min + 1;
+					zone->free_pages < zone->pages_min)) {
+				sum += zone->pages_min;
 				sum -= zone->free_pages;
 				sum -= zone->inactive_clean_pages;
 			}
@@ -777,38 +806,46 @@
 }
 
 /*
- * We need to make the locks finer granularity, but right
- * now we need this so that we can do page allocations
- * without holding the kernel lock etc.
+ * Refill_inactive is the function used to scan and age the pages on
+ * the active list and in the working set of processes, moving the
+ * little-used pages to the inactive list.
  *
- * We want to try to free "count" pages, and we want to 
- * cluster them so that we get good swap-out behaviour.
+ * When called by kswapd, we try to deactivate as many pages as needed
+ * to recover from the inactive page shortage. This makes it possible
+ * for kswapd to keep up with memory demand so user processes can get
+ * low latency on memory allocations.
  *
- * OTOH, if we're a user process (and not kswapd), we
- * really care about latency. In that case we don't try
- * to free too many pages.
+ * However, when the system starts to get overloaded we can get called
+ * by user processes. For user processes we want to both reduce the
+ * latency and make sure that multiple user processes together don't
+ * deactivate too many pages. To achieve this we simply do less work
+ * when called from a user process.
  */
 #define DEF_PRIORITY (6)
 static int refill_inactive(unsigned int gfp_mask, int user)
 {
 	int count, start_count, maxtry;
 
-	count = inactive_shortage() + free_shortage();
-	if (user)
+	if (user) {
 		count = (1 << page_cluster);
-	start_count = count;
+		maxtry = 6;
+	} else {
+		count = inactive_shortage();
+		maxtry = 1 << DEF_PRIORITY;
+	}
 
-	maxtry = 6;
+	start_count = count;
 	do {
 		if (current->need_resched) {
 			__set_current_state(TASK_RUNNING);
 			schedule();
+			if (!inactive_shortage())
+				return 1;
 		}
 
-		while (refill_inactive_scan(DEF_PRIORITY, 1)) {
-			if (--count <= 0)
-				goto done;
-		}
+		count -= refill_inactive_scan(DEF_PRIORITY, count);
+		if (count <= 0)
+			goto done;
 
 		/* If refill_inactive_scan failed, try to page stuff out.. */
 		swap_out(DEF_PRIORITY, gfp_mask);
@@ -834,9 +871,11 @@
 	 * before we get around to moving them to the other
 	 * list, so this is a relatively cheap operation.
 	 */
-	if (free_shortage() || nr_inactive_dirty_pages > nr_free_pages() +
-			nr_inactive_clean_pages())
+	if (free_shortage()) {
 		ret += page_launder(gfp_mask, user);
+		shrink_dcache_memory(DEF_PRIORITY, gfp_mask);
+		shrink_icache_memory(DEF_PRIORITY, gfp_mask);
+	}
 
 	/*
 	 * If needed, we move pages from the active list
@@ -846,30 +885,15 @@
 		ret += refill_inactive(gfp_mask, user);
 
 	/* 	
-	 * Delete pages from the inode and dentry caches and 
-	 * reclaim unused slab cache if memory is low.
+	 * Reclaim unused slab cache if memory is low.
 	 */
-	if (free_shortage()) {
-		shrink_dcache_memory(DEF_PRIORITY, gfp_mask);
-		shrink_icache_memory(DEF_PRIORITY, gfp_mask);
-	} else {
-		/*
-		 * Illogical, but true. At least for now.
-		 *
-		 * If we're _not_ under shortage any more, we
-		 * reap the caches. Why? Because a noticeable
-		 * part of the caches are the buffer-heads, 
-		 * which we'll want to keep if under shortage.
-		 */
-		kmem_cache_reap(gfp_mask);
-	} 
+	kmem_cache_reap(gfp_mask);
 
 	return ret;
 }
 
 DECLARE_WAIT_QUEUE_HEAD(kswapd_wait);
 DECLARE_WAIT_QUEUE_HEAD(kswapd_done);
-struct task_struct *kswapd_task;
 
 /*
  * The background pageout daemon, started as a kernel thread
@@ -888,11 +912,9 @@
 {
 	struct task_struct *tsk = current;
 
-	tsk->session = 1;
-	tsk->pgrp = 1;
+	daemonize();
 	strcpy(tsk->comm, "kswapd");
 	sigfillset(&tsk->blocked);
-	kswapd_task = tsk;
 	
 	/*
 	 * Tell the memory management that we're a "memory allocator",
@@ -918,18 +940,15 @@
 		if (inactive_shortage() || free_shortage()) 
 			do_try_to_free_pages(GFP_KSWAPD, 0);
 
-		/*
-		 * Do some (very minimal) background scanning. This
-		 * will scan all pages on the active list once
-		 * every minute. This clears old referenced bits
-		 * and moves unused pages to the inactive list.
-		 */
-		refill_inactive_scan(DEF_PRIORITY, 0);
-
-		/* Once a second, recalculate some VM stats. */
+		/* Once a second ... */
 		if (time_after(jiffies, recalc + HZ)) {
 			recalc = jiffies;
+
+			/* Recalculate VM statistics. */
 			recalculate_vm_stats();
+
+			/* Do background page aging. */
+			refill_inactive_scan(DEF_PRIORITY, 0);
 		}
 
 		run_task_queue(&tq_disk);
@@ -964,8 +983,8 @@
 
 void wakeup_kswapd(void)
 {
-	if (current != kswapd_task)
-		wake_up_process(kswapd_task);
+	if (waitqueue_active(&kswapd_wait))
+		wake_up_interruptible(&kswapd_wait);
 }
 
 /*
@@ -997,8 +1016,7 @@
 	struct task_struct *tsk = current;
 	pg_data_t *pgdat;
 
-	tsk->session = 1;
-	tsk->pgrp = 1;
+	daemonize();
 	strcpy(tsk->comm, "kreclaimd");
 	sigfillset(&tsk->blocked);
 	current->flags |= PF_MEMALLOC;