libc: wunlock() part 2

the initial issue was that wunlock() would wakeup readers while
holding the spinlock causing deadlock in libthread programs where
rendezvous() would do a thread switch within the same process
which then can acquire the RWLock again.

the first fix tried to prevent holding the spinlock, waking up
one reader at a time with releasing an re-acquiering the spinlock.
this violates the invariant that readers can only wakup writers
in runlock() when multiple readers where queued at the time of
wunlock(). at the first wakeup, q->head != nil so runlock() would
find a reader queued on runlock() when it expected a writer.

this (hopefully last) fix unlinks *all* the reader QLp's atomically
and in order while holding the spinlock and then traverses the
dequeued chain of QLp structures again to call rendezvous() so
the invariant described above holds.

1 files changed, 22 insertions, 15 deletions

diff --git a/sys/src/libc/9sys/qlock.c b/sys/src/libc/9sys/qlock.c
index 44cce8ab8..e968080be 100644
--- a/sys/src/libc/9sys/qlock.c
+++ b/sys/src/libc/9sys/qlock.c
@@ -175,8 +175,8 @@ runlock(RWLock *q)
 	if(p->state != QueuingW)
 		abort();
 	q->head = p->next;
-	if(q->head == 0)
-		q->tail = 0;
+	if(q->head == nil)
+		q->tail = nil;
 	q->writer = 1;
 	unlock(&q->lock);
 
@@ -233,7 +233,7 @@ canwlock(RWLock *q)
 void
 wunlock(RWLock *q)
 {
-	QLp *p;
+	QLp *p, *x;
 
 	lock(&q->lock);
 	if(q->writer == 0)
@@ -254,24 +254,31 @@ wunlock(RWLock *q)
 			;
 		return;
 	}
-
 	if(p->state != QueuingR)
 		abort();
 
-	q->writer = 0;
-	do {
-		/* wake waiting readers */
-		q->head = p->next;
-		if(q->head == nil)
-			q->tail = nil;
+	/* collect waiting readers */
+	q->readers = 1;
+	for(x = p->next; x != nil && x->state == QueuingR; x = x->next){
 		q->readers++;
-		unlock(&q->lock);
+		p = x;
+	}
+	p->next = nil;
+	p = q->head;
+
+	/* queue remaining writers */
+	q->head = x;
+	if(x == nil)
+		q->tail = nil;
+	q->writer = 0;
+	unlock(&q->lock);
+
+	/* wakeup waiting readers */
+	for(; p != nil; p = x){
+		x = p->next;
 		while((*_rendezvousp)(p, 0) == (void*)~0)
 			;
-		lock(&q->lock);
-		p = q->head;
-	} while(p != nil && p->state == QueuingR && q->writer == 0);
-	unlock(&q->lock);
+	}
 }
 
 void