kernel: avoid palloc lock during mmurelease()

Previously, mmurelease() was always called with
palloc spinlock held.

This is unneccesary for some mmurelease()
implementations as they wont release pages
to the palloc pool.

This change removes pagechainhead() and
pagechaindone() and replaces them with just
freepages() call, which aquires the palloc
lock internally as needed.

freepages() avoids holding the palloc lock
while walking the linked list of pages,
avoding some lock contention.

13 files changed, 82 insertions, 162 deletions

diff --git a/sys/src/9/bcm/mmu.c b/sys/src/9/bcm/mmu.c
index 864757079..655d847ae 100644
--- a/sys/src/9/bcm/mmu.c
+++ b/sys/src/9/bcm/mmu.c
@@ -173,17 +173,9 @@ flushmmu(void)
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %lud", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
+
+	freepages(proc->mmul2cache, nil, 0);
 	proc->mmul2cache = nil;
 
 	mmul1empty();
diff --git a/sys/src/9/bcm64/mmu.c b/sys/src/9/bcm64/mmu.c
index 4c664554b..8538369fa 100644
--- a/sys/src/9/bcm64/mmu.c
+++ b/sys/src/9/bcm64/mmu.c
@@ -518,20 +518,10 @@ mmuswitch(Proc *p)
 void
 mmurelease(Proc *p)
 {
-	Page *t;
-
 	mmuswitch(nil);
 	mmufree(p);
-
-	if((t = p->mmufree) != nil){
-		do {
-			p->mmufree = t->next;
-			if(--t->ref != 0)
-				panic("mmurelease: bad page ref");
-			pagechainhead(t);
-		} while((t = p->mmufree) != nil);
-		pagechaindone();
-	}
+	freepages(p->mmufree, nil, 0);
+	p->mmufree = nil;
 }
 
 void
diff --git a/sys/src/9/cycv/mmu.c b/sys/src/9/cycv/mmu.c
index 8a449e70b..213a88178 100644
--- a/sys/src/9/cycv/mmu.c
+++ b/sys/src/9/cycv/mmu.c
@@ -197,23 +197,21 @@ flushmmu(void)
 void
 mmurelease(Proc *proc)
 {
-	Page *p, *n;
+	Page *p;
 
-	if(islo())
-		panic("mmurelease: islo");
-	
 	l1switch(&m->l1, 0);
-	if(proc->kmaptable != nil){
+	if((p = proc->kmaptable) != nil){
+		if(p->ref != 1)
+			panic("mmurelease: kmap ref %ld", p->ref);
 		if(proc->l1 == nil)
 			panic("mmurelease: no l1");
-		if(decref(proc->kmaptable) != 0)
-			panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
 		if(proc->nkmap)
 			panic("mmurelease: nkmap %d", proc->nkmap);
-		if(PPN(proc->l1->va[L1X(KMAP)]) != proc->kmaptable->pa)
-			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], proc->kmaptable->pa);
+		if(PPN(proc->l1->va[L1X(KMAP)]) != p->pa)
+			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], p->pa);
 		proc->l1->va[L1X(KMAP)] = 0;
-		pagechainhead(proc->kmaptable);
+		p->next = proc->mmufree;
+		proc->mmufree = p;
 		proc->kmaptable = nil;
 	}
 	if(proc->l1 != nil){
@@ -221,14 +219,7 @@ mmurelease(Proc *proc)
 		l1free(proc->l1);
 		proc->l1 = nil;
 	}
-	for(p = proc->mmufree; p != nil; p = n){
-		n = p->next;
-		if(decref(p) != 0)
-			panic("mmurelease: p->ref %ld", p->ref);
-		pagechainhead(p);
-	}
-	if(proc->mmufree != nil)
-		pagechaindone();
+	freepages(proc->mmufree, nil, 0);
 	proc->mmufree = nil;
 }
 
diff --git a/sys/src/9/kw/mmu.c b/sys/src/9/kw/mmu.c
index bab4238fd..0696bff35 100644
--- a/sys/src/9/kw/mmu.c
+++ b/sys/src/9/kw/mmu.c
@@ -252,20 +252,12 @@ flushmmu(void)
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	/* write back dirty and invalidate l1 caches */
 	cacheuwbinv();
 
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %lud", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
+
+	freepages(proc->mmul2cache, nil, 0);
 	proc->mmul2cache = nil;
 
 	mmul1empty();
diff --git a/sys/src/9/omap/mmu.c b/sys/src/9/omap/mmu.c
index d427f1d76..12383c962 100644
--- a/sys/src/9/omap/mmu.c
+++ b/sys/src/9/omap/mmu.c
@@ -234,20 +234,12 @@ flushmmu(void)
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	/* write back dirty and invalidate l1 caches */
 	cacheuwbinv();
 
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %ld", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
+
+	freepages(proc->mmul2cache, nil, 0);
 	proc->mmul2cache = nil;
 
 	mmul1empty();
diff --git a/sys/src/9/pc/mmu.c b/sys/src/9/pc/mmu.c
index df0c23520..a21c94197 100644
--- a/sys/src/9/pc/mmu.c
+++ b/sys/src/9/pc/mmu.c
@@ -320,54 +320,48 @@ mmuswitch(Proc* proc)
  *   cleaning any user entries in the pdb (proc->mmupdb);
  *   if there's a pdb put it in the cache of pre-initialised pdb's
  *   for this processor (m->pdbpool) or on the process' free list;
- *   finally, place any pages freed back into the free pool (palloc).
- * This routine is only called from schedinit() with palloc locked.
+ *   finally, place any pages freed back into the free pool (freepages).
  */
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
 	ulong *pdb;
+	Page *page;
 
-	if(islo())
-		panic("mmurelease: islo");
 	taskswitch(PADDR(m->pdb), (ulong)m + BY2PG);
-	if(proc->kmaptable != nil){
+	if((page = proc->kmaptable) != nil){
+		if(page->ref != 1)
+			panic("mmurelease: kmap ref %ld", page->ref);
 		if(proc->mmupdb == nil)
 			panic("mmurelease: no mmupdb");
-		if(--proc->kmaptable->ref != 0)
-			panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
 		if(proc->nkmap)
 			panic("mmurelease: nkmap %d", proc->nkmap);
 		/*
 		 * remove kmaptable from pdb before putting pdb up for reuse.
 		 */
 		pdb = tmpmap(proc->mmupdb);
-		if(PPN(pdb[PDX(KMAP)]) != proc->kmaptable->pa)
+		if(PPN(pdb[PDX(KMAP)]) != page->pa)
 			panic("mmurelease: bad kmap pde %#.8lux kmap %#.8lux",
-				pdb[PDX(KMAP)], proc->kmaptable->pa);
+				pdb[PDX(KMAP)], page->pa);
 		pdb[PDX(KMAP)] = 0;
 		tmpunmap(pdb);
+
 		/*
 		 * move kmaptable to free list.
 		 */
-		pagechainhead(proc->kmaptable);
+		page->next = proc->mmufree;
+		proc->mmufree = page;
 		proc->kmaptable = nil;
 	}
-	if(proc->mmupdb != nil){
+	if((page = proc->mmupdb) != nil){
 		mmuptefree(proc);
-		mmupdbfree(proc, proc->mmupdb);
+		mmupdbfree(proc, page);
 		proc->mmupdb = nil;
 	}
-	for(page = proc->mmufree; page != nil; page = next){
-		next = page->next;
-		if(--page->ref != 0)
-			panic("mmurelease: page->ref %ld", page->ref);
-		pagechainhead(page);
+	if((page = proc->mmufree) != nil){
+		freepages(page, nil, 0);
+		proc->mmufree = nil;
 	}
-	if(proc->mmufree != nil)
-		pagechaindone();
-	proc->mmufree = nil;
 	if(proc->ldt != nil){
 		free(proc->ldt);
 		proc->ldt = nil;
diff --git a/sys/src/9/port/devsegment.c b/sys/src/9/port/devsegment.c
index fcaf8f3e4..dfbd34fbf 100644
--- a/sys/src/9/port/devsegment.c
+++ b/sys/src/9/port/devsegment.c
@@ -469,7 +469,7 @@ fixedseg(uintptr va, ulong len)
 {
 	KMap *k;
 	Segment *s;
-	Page **f, *p, *l, *h;
+	Page **f, *p, *l, *h, *t;
 	ulong n, i;
 	int color;
 
@@ -492,12 +492,13 @@ fixedseg(uintptr va, ulong len)
 			continue;
 
 		i = 0;
-		h = nil;
+		h = t = nil;
 		f = &palloc.head;
 		while((p = *f) != nil){
 			if(p > &l[-len] && p <= l){
 				*f = p->next;
-				p->next = h;
+				if((p->next = h) == nil)
+					t = p;
 				h = p;
 				if(++i < len)
 					continue;
@@ -505,15 +506,15 @@ fixedseg(uintptr va, ulong len)
 			}
 			f = &p->next;
 		}
-		palloc.freecount -= i;
 
 		if(i != len){
-			while((p = h) != nil){
-				h = h->next;
-				pagechainhead(p);
+			if(h != nil){
+				t->next = palloc.head;
+				palloc.head = h;
 			}
 			goto Retry;
 		}
+		palloc.freecount -= i;
 		unlock(&palloc);
 
 		p = &l[-len];
diff --git a/sys/src/9/port/page.c b/sys/src/9/port/page.c
index 0a039c099..b7194058e 100644
--- a/sys/src/9/port/page.c
+++ b/sys/src/9/port/page.c
@@ -11,7 +11,7 @@ void
 pageinit(void)
 {
 	int color, i, j;
-	Page *p;
+	Page *p, **t;
 	Pallocmem *pm;
 	vlong m, v, u;
 
@@ -29,8 +29,12 @@ pageinit(void)
 	}
 
 	color = 0;
+	palloc.freecount = 0;
 	palloc.head = nil;
+
+	t = &palloc.head;
 	p = palloc.pages;
+
 	for(i=0; i<nelem(palloc.mem); i++){
 		pm = &palloc.mem[i];
 		for(j=0; j<pm->npage; j++){
@@ -40,7 +44,8 @@ pageinit(void)
 				continue;
 			p->color = color;
 			color = (color+1)%NCOLOR;
-			pagechainhead(p);
+			*t = p, t = &p->next;
+			palloc.freecount++;
 			p++;
 		}
 	}
@@ -65,15 +70,7 @@ pageinit(void)
 	print("%lldM swap\n", v/(1024*1024));
 }
 
-void
-pagechainhead(Page *p)
-{
-	p->next = palloc.head;
-	palloc.head = p;
-	palloc.freecount++;
-}
-
-void
+static void
 pagechaindone(void)
 {
 	if(palloc.pwait[0].p != nil && wakeup(&palloc.pwait[0]) != nil)
@@ -85,11 +82,23 @@ pagechaindone(void)
 void
 freepages(Page *head, Page *tail, ulong np)
 {
-	assert(palloc.Lock.p == up);
+	if(head == nil)
+		return;
+	if(tail == nil){
+		tail = head;
+		for(np = 1;; np++){
+			tail->ref = 0;
+			if(tail->next == nil)
+				break;
+			tail = tail->next;
+		}
+	}
+	lock(&palloc);
 	tail->next = palloc.head;
 	palloc.head = head;
 	palloc.freecount += np;
 	pagechaindone();
+	unlock(&palloc);
 }
 
 ulong
@@ -138,11 +147,8 @@ pagereclaim(Image *i)
 	}
 	putimage(i);
 
-	if(np > 0){
-		lock(&palloc);
+	if(np > 0)
 		freepages(fh, ft, np);
-		unlock(&palloc);
-	}
 
 	return np;
 }
@@ -237,11 +243,8 @@ putpage(Page *p)
 		decref(p);
 		return;
 	}
-	if(decref(p) == 0){
-		lock(&palloc);
+	if(decref(p) == 0)
 		freepages(p, p, 1);
-		unlock(&palloc);
-	}
 }
 
 void
diff --git a/sys/src/9/port/portfns.h b/sys/src/9/port/portfns.h
index d1d2c485c..7446513be 100644
--- a/sys/src/9/port/portfns.h
+++ b/sys/src/9/port/portfns.h
@@ -218,8 +218,6 @@ int		okaddr(uintptr, ulong, int);
 int		openmode(ulong);
 Block*		packblock(Block*);
 Block*		padblock(Block*, int);
-void		pagechaindone(void);
-void		pagechainhead(Page*);
 void		pageinit(void);
 ulong		pagereclaim(Image*);
 void		panic(char*, ...);
diff --git a/sys/src/9/port/proc.c b/sys/src/9/port/proc.c
index a51232d44..567ab0756 100644
--- a/sys/src/9/port/proc.c
+++ b/sys/src/9/port/proc.c
@@ -81,27 +81,21 @@ schedinit(void)		/* never returns */
 		case Moribund:
 			up->state = Dead;
 			edfstop(up);
-			if(up->edf != nil)
+			if(up->edf != nil){
 				free(up->edf);
-			up->edf = nil;
+				up->edf = nil;
+			}
 
-			/*
-			 * Holding locks from pexit:
-			 * 	procalloc
-			 *	palloc
-			 */
 			mmurelease(up);
-			unlock(&palloc);
 
-			updatecpu(up);
+			lock(&procalloc);
 			up->mach = nil;
-
 			up->qnext = procalloc.free;
 			procalloc.free = up;
-
 			/* proc is free now, make sure unlock() wont touch it */
 			up = procalloc.Lock.p = nil;
 			unlock(&procalloc);
+
 			sched();
 		}
 		coherence();
@@ -1223,10 +1217,6 @@ pexit(char *exitstr, int freemem)
 	}
 	qunlock(&up->seglock);
 
-	/* Sched must not loop for these locks */
-	lock(&procalloc);
-	lock(&palloc);
-
 	edfstop(up);
 	up->state = Moribund;
 	sched();
diff --git a/sys/src/9/teg2/mmu.c b/sys/src/9/teg2/mmu.c
index 871e1c1c3..6005e5065 100644
--- a/sys/src/9/teg2/mmu.c
+++ b/sys/src/9/teg2/mmu.c
@@ -475,20 +475,12 @@ flushmmu(void)
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	/* write back dirty and invalidate caches */
 	l1cache->wbinv();
 
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %ld", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
+
+	freepages(proc->mmul2cache, nil, 0);
 	proc->mmul2cache = nil;
 
 	mmul1empty();
diff --git a/sys/src/9/xen/mmu.c b/sys/src/9/xen/mmu.c
index 1a5c3d067..250d08371 100644
--- a/sys/src/9/xen/mmu.c
+++ b/sys/src/9/xen/mmu.c
@@ -282,15 +282,8 @@ mmurelease(Proc* proc)
 		}
 	}
 
-	for(page = proc->mmufree; page; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %ld\n", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmufree)
-		pagechaindone();
-	proc->mmufree = 0;
+	freepages(proc->mmufree, nil, 0);
+	proc->mmufree = nil;
 }
 
 static Page*
diff --git a/sys/src/9/zynq/mmu.c b/sys/src/9/zynq/mmu.c
index ed6412e70..9b4c67d4c 100644
--- a/sys/src/9/zynq/mmu.c
+++ b/sys/src/9/zynq/mmu.c
@@ -205,23 +205,22 @@ flushmmu(void)
 void
 mmurelease(Proc *proc)
 {
-	Page *p, *n;
+	Page *p;
 
-	if(islo())
-		panic("mmurelease: islo");
-	
 	l1switch(&m->l1, 0);
-	if(proc->kmaptable != nil){
+	if((p = proc->kmaptable) != nil){
+		if(p->ref != 1)
+			panic("mmurelease: kmap ref %ld", p->ref);
 		if(proc->l1 == nil)
 			panic("mmurelease: no l1");
-		if(decref(proc->kmaptable) != 0)
-			panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
 		if(proc->nkmap)
 			panic("mmurelease: nkmap %d", proc->nkmap);
-		if(PPN(proc->l1->va[L1X(KMAP)]) != proc->kmaptable->pa)
-			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], proc->kmaptable->pa);
+		if(PPN(proc->l1->va[L1X(KMAP)]) != p->pa)
+			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], p->pa);
 		proc->l1->va[L1X(KMAP)] = 0;
-		pagechainhead(proc->kmaptable);
+
+		p->next = proc->mmufree;
+		proc->mmufree = p;
 		proc->kmaptable = nil;
 	}
 	if(proc->l1 != nil){
@@ -229,14 +228,7 @@ mmurelease(Proc *proc)
 		l1free(proc->l1);
 		proc->l1 = nil;
 	}
-	for(p = proc->mmufree; p != nil; p = n){
-		n = p->next;
-		if(decref(p) != 0)
-			panic("mmurelease: p->ref %ld", p->ref);
-		pagechainhead(p);
-	}
-	if(proc->mmufree != nil)
-		pagechaindone();
+	freepages(proc->mmufree, nil, 0);
 	proc->mmufree = nil;
 }