shithub: vmxsmp

Download patch

ref: 2c9eeaf3cfd204ee49d676e8716445bb563249cd
parent: a3cfb13960d937a0ecaad3ea738fd55bf7c14105
author: glenda <glenda@fileserver>
date: Mon Dec 29 23:06:24 EST 2025 

code cleanup, pci improvement, proper shutdown


--- a/README
+++ b/README
@@ -1,3 +1,51 @@
+** INSTALL ** 
+
+-> cp devvmx.c /sys/src/9/pc/devvmx.c
+-> Recompile the kernel and reboot:
+
+cd /sys/src/9/pc64
+mk clean
+mk 
+mk install
+bind -a '#S' /dev
+9fs 9fat
+cp /n/9fat/9pc64 /n/9fat/oldpc64 # backup 
+cp /amd64/9pc64 /n/9fat/9pc64
+fshalt -r
+
+-> Build vmx:
+
+mk
+mk install
+
+-> Install alpine linux, 
+
+./runvm alpineinstall
+
+You should see 4 cores in /proc/cpuinfo
+
+-> Use alpine linux, 
+
+./runvm alpinerun
+
+-> These scripts assume the existence of /net/ether0
+
+** NEW FEATURES **
+
+-> SMP, to specify number cores use -x option in vmx, by default 4 cores.
+
+-> Faster vioblk, on my laptop I can get up to 240Mb/s vs the 
+previous 180Mb/s . Presumably vionet is also faster. 
+
+** BUGS **
+
+-> OpenBSD is slow
+-> 9front is slow when two disks are attached
+-> Mouse is slow
+-> Cannot boot with more than 4G (trivial to fix though)
+
+** (PREVIOUS) NOTES **
+
 -> In devvmx.c we add two features, setting the tsc offset and VMX PREEMPT. Therefore,
 
 **Running this requires recompiling the kernel with the provided devvmx.c file**
--- a/acpi.c
+++ b/acpi.c
@@ -429,8 +429,7 @@
     memcpy(p + 36, aml, aml_len);
 
     p[9] = acpi_checksum(p, len);
-	if (debug)
-	    fprint(2, "build_dsdt: aml_len=%d total=%d\n", aml_len, len);
+	dprint("build_dsdt: aml_len=%d total=%d\n", aml_len, len);
 
     return len;
 }
@@ -679,13 +678,13 @@
 	rsdt_len = build_rsdt(rsdt);
 	rsdp_len = build_rsdp(rsdp);
 	
-	vmerror("ACPI tables created:");
-	vmerror("  RSDP at %#x (%d bytes)", RSDP_ADDR, rsdp_len);
-	vmerror("  RSDT at %#x (%d bytes)", RSDT_ADDR, rsdt_len);
-	vmerror("  MADT at %#x (%d bytes) - %d CPUs", MADT_ADDR, madt_len, nvcpu);
-	vmerror("  FADT at %#x (%d bytes)", FADT_ADDR, fadt_len);
-	vmerror("  FACS at %#x (%d bytes)", FACS_ADDR, facs_len);
-	vmerror("  DSDT at %#x (%d bytes)", DSDT_ADDR, dsdt_len);
-	vmerror("  HPET at %#x (%d bytes)", HPET_ADDR, hpet_len);
+	dprint("ACPI tables created:");
+	dprint("  RSDP at %#x (%d bytes)", RSDP_ADDR, rsdp_len);
+	dprint("  RSDT at %#x (%d bytes)", RSDT_ADDR, rsdt_len);
+	dprint("  MADT at %#x (%d bytes) - %d CPUs", MADT_ADDR, madt_len, nvcpu);
+	dprint("  FADT at %#x (%d bytes)", FADT_ADDR, fadt_len);
+	dprint("  FACS at %#x (%d bytes)", FACS_ADDR, facs_len);
+	dprint("  DSDT at %#x (%d bytes)", DSDT_ADDR, dsdt_len);
+	dprint("  HPET at %#x (%d bytes)", HPET_ADDR, hpet_len);
 }
 
--- /dev/null
+++ b/cleanup.c
@@ -1,0 +1,213 @@
+/* cleanup.c - VMX SMP cleanup handling */
+
+#include <u.h>
+#include <libc.h>
+#include <thread.h>
+#include "dat.h"
+#include "fns.h"
+
+static int parent_pid;
+static int child_pids[MAXVCPU];
+static int nchildren;
+
+extern int ctlfd, regsfd, mapfd, waitfd;
+extern int kbdpipe[2];
+extern int mousepipe[2];
+extern int uartpipe[2][2];
+extern int uarttxpipe[2][2];
+extern int wakepipe[MAXVCPU][2];
+extern int hltpipe[MAXVCPU][2];
+
+/*
+ * Stop and release VMX context - each process does this for its own
+ */
+static void
+cleanup_vmx(void)
+{
+
+	extern IOApic *ioapic;
+	extern IpiQueue *ipiqueue;
+	extern PciShared *pcishared;
+	extern VIOShared *vioshared;
+
+    /* Detach shared segments first */
+    if(ioapic != nil){ segdetach(ioapic); ioapic = nil; }
+    if(ipiqueue != nil){ segdetach(ipiqueue); ipiqueue = nil; }
+    if(pcishared != nil){ segdetach(pcishared); pcishared = nil; }
+    if(vioshared != nil){ segdetach(vioshared); vioshared = nil; }
+
+    if(ctlfd >= 0){
+        write(ctlfd, "stop", 4);
+        write(ctlfd, "quit", 4);
+    }
+    
+    if(waitfd >= 0){ close(waitfd); waitfd = -1; }
+    if(mapfd >= 0){ close(mapfd); mapfd = -1; }
+    if(regsfd >= 0){ close(regsfd); regsfd = -1; }
+    if(ctlfd >= 0){ close(ctlfd); ctlfd = -1; }
+
+	extern char name[128];
+
+	char buf[128];
+	snprint(buf, sizeof(buf), "#X/%s/ctl", name);
+	remove(buf);
+	snprint(buf, sizeof(buf), "#X/%s", name);
+	remove(buf);
+
+}
+
+/*
+ * Remove shared segments - parent only
+ */
+static void
+cleanup_shared(void)
+{
+    if(vmx_segprefix == nil)
+        return;
+    
+    rmseg("ioapic");
+    rmseg("ipi");
+    rmseg("pci");
+    rmseg("vio");
+    
+    free(vmx_segprefix);
+    vmx_segprefix = nil;
+}
+
+/*
+ * Close all pipes - parent only
+ */
+static void
+cleanup_pipes(void)
+{
+    int i;
+    
+    for(i = 0; i < MAXVCPU; i++){
+        if(wakepipe[i][0] >= 0) close(wakepipe[i][0]);
+        if(wakepipe[i][1] >= 0) close(wakepipe[i][1]);
+        if(hltpipe[i][0] >= 0) close(hltpipe[i][0]);
+        if(hltpipe[i][1] >= 0) close(hltpipe[i][1]);
+        wakepipe[i][0] = wakepipe[i][1] = -1;
+        hltpipe[i][0] = hltpipe[i][1] = -1;
+    }
+    
+    if(kbdpipe[0] >= 0) close(kbdpipe[0]);
+    if(kbdpipe[1] >= 0) close(kbdpipe[1]);
+    kbdpipe[0] = kbdpipe[1] = -1;
+    
+    if(mousepipe[0] >= 0) close(mousepipe[0]);
+    if(mousepipe[1] >= 0) close(mousepipe[1]);
+    mousepipe[0] = mousepipe[1] = -1;
+    
+    for(i = 0; i < 2; i++){
+        if(uartpipe[i][0] >= 0) close(uartpipe[i][0]);
+        if(uartpipe[i][1] >= 0) close(uartpipe[i][1]);
+        if(uarttxpipe[i][0] >= 0) close(uarttxpipe[i][0]);
+        if(uarttxpipe[i][1] >= 0) close(uarttxpipe[i][1]);
+        uartpipe[i][0] = uartpipe[i][1] = -1;
+        uarttxpipe[i][0] = uarttxpipe[i][1] = -1;
+    }
+}
+
+/*
+ * Kill all child CPU processes
+ */
+static void
+killchildren(void)
+{
+    int i;
+    Waitmsg *w;
+    
+    for(i = 0; i < nchildren; i++){
+        if(child_pids[i] > 0)
+            postnote(PNPROC, child_pids[i], "kill");
+    }
+    
+    /* Actually wait for them to exit */
+    for(i = 0; i < nchildren; i++){
+        if(child_pids[i] > 0){
+            w = wait();
+            if(w != nil)
+                free(w);
+        }
+    }
+}
+
+/*
+ * Full cleanup - called on exit
+ */
+ 
+void
+vmx_cleanup(void)
+{
+    int isparent = (getpid() == parent_pid);
+    
+    dprint("vmx_cleanup: pid=%d isparent=%d\n", getpid(), isparent);
+    
+    if(isparent)
+        killchildren();  /* This should sleep(100) internally */
+    
+    cleanup_vmx();
+    
+    if(isparent){
+        sleep(200);      /* Extra wait for children to fully exit */
+        cleanup_pipes();
+        cleanup_shared();
+	 
+    }
+}
+
+/*
+ * atexit handler
+ */
+static void
+vmx_atexit(void)
+{
+    vmx_cleanup();
+}
+
+/*
+ * Note handler - catch Del, kill, hangup
+ */  
+int
+vmx_notehandler(void *, char *note)
+{
+    dprint("NOTE: %s (pid %d)\n", note, getpid());
+    
+    if(strcmp(note, "interrupt") == 0 ||
+       strcmp(note, "kill") == 0 ||
+       strcmp(note, "hangup") == 0){
+        
+        vmx_cleanup();
+        
+	 
+ 
+    }
+    return 0;  /* Note not handled, pass to default */
+}
+
+/*
+ * Register cleanup handlers - call early in threadmain()
+ */
+void
+vmx_cleanup_init(void)
+{
+
+    parent_pid = getpid();
+    nchildren = 0;
+    memset(child_pids, 0, sizeof(child_pids));
+    /* Use threadnotify for libthread programs */
+    threadnotify(vmx_notehandler, 1);
+    
+    atexit(vmx_atexit);
+}
+
+/*
+ * Register a child process - call from forkcpu() in parent
+ */
+void
+vmx_register_child(int pid)
+{
+    if(nchildren < MAXVCPU)
+        child_pids[nchildren++] = pid;
+}
--- a/dat.h
+++ b/dat.h
@@ -6,6 +6,9 @@
 
 #define BACKSTOP_NS  50000000LL  /* 10ms */
 
+extern void dprint(char *fmt, ...);
+
+extern char *vmx_segprefix;  
  
 #define MOUSE_PKT_BUF_SIZE 256
 
@@ -357,6 +360,14 @@
 struct VIOShared {
     u8int data[32768];
     uintptr alloc;
+ 	u8int isrstat;  
+
+    int devpipes[8][2];
+    int ndevpipes;
+    void *blkdevs[8];    /* ADD: VIODev pointers for block devices */
+    int nblkdevs;         /* ADD */
+    void *netdevs[8];    /* ADD: VIODev pointers for net devices */  
+    int nnetdevs;         /* ADD */
 };
 extern VIOShared *vioshared;
 
--- a/exith.c
+++ b/exith.c
@@ -358,8 +358,7 @@
 	u8int *ip = gptr(phys, 16);
 
 	if(ip == nil){
-		if (debug)
-			fprint(2, "setmovdest: gptr failed rip=%#llx phys=%#llx\n", rip, phys);
+		dprint("setmovdest: gptr failed rip=%#llx phys=%#llx\n", rip, phys);
 		rset(RAX, val);
 		return 0;
 	}
@@ -366,8 +365,7 @@
 	
 	/* Debug: show what we're decoding */
 	if(ip[0] == 0 && ip[1] == 0 && ip[2] == 0 && ip[3] == 0){
-		if (debug)
-			fprint(2, "setmovdest: zero opcodes! rip=%#llx phys=%#llx\n", rip, phys);
+		dprint("setmovdest: zero opcodes! rip=%#llx phys=%#llx\n", rip, phys);
 	}
 
 	int i = 0;
@@ -639,9 +637,7 @@
 	}
 
 	/* Fallback - log and try to handle common patterns */
-	if (debug)
-		fprint(2, "setmovdest: unhandled opcode %02x %02x %02x %02x (rex=%02x has66=%d) val=%#llx\n",
-	       ip[i], ip[i+1], ip[i+2], ip[i+3], rex, has66, val);  /* Changed %#x to %#llx */
+	dprint("setmovdest: unhandled opcode %02x %02x %02x %02x (rex=%02x has66=%d) val=%#llx\n", ip[i], ip[i+1], ip[i+2], ip[i+3], rex, has66, val);  /* Changed %#x to %#llx */
 	
 	/*
 	 * For unhandled read instructions, try to determine destination from ModRM.
@@ -653,8 +649,7 @@
 		int modrm = ip[i+1];
 		int reg = (modrm >> 3) & 7;
 		if(rex & 4) reg |= 8;
-		if (debug)
-			fprint(2, "setmovdest: fallback using reg %d (%s)\n", reg, x86reg[reg]);
+		dprint("setmovdest: fallback using reg %d (%s)\n", reg, x86reg[reg]);
 		rset(x86reg[reg], val);
 		return 1;
 	}
@@ -673,38 +668,12 @@
 void
 ipiqueueinit(void)
 {
-	int fd;
-	char buf[64];
-	
-	remove("#g/vmx.ipi/ctl");
-	remove("#g/vmx.ipi");
-
-	/* Create shared segment */
-	fd = create("#g/vmx.ipi", OREAD, DMDIR | 0777);
-	if(fd >= 0) close(fd);
-	
-	fd = open("#g/vmx.ipi/ctl", OWRITE);
-	if(fd < 0){
-		fprint(2, "ipiqueueinit: cannot open ctl: %r\n");
-		return;
-	}
-	
-	snprint(buf, sizeof buf, "va 0x300001000 0x1000 sticky");
-
-	write(fd, buf, strlen(buf));
-	close(fd);
-	
-	ipiqueue = segattach(0, "vmx.ipi", nil, 0x1000);
-	if(ipiqueue == (void*)-1){
-		fprint(2, "ipiqueueinit: segattach failed: %r\n");
-		ipiqueue = nil;
-		return;
-	}
-	
-	memset(ipiqueue, 0, sizeof(IpiQueue));
+	ipiqueue = mkseg("ipi", 0x300001000, 0x1000);
+    memset(ipiqueue, 0, sizeof(IpiQueue));
+	 
 	ipiqueue->pids[0] = getpid();
 	
-	fprint(2, "IPI queue initialized\n");
+	dprint("IPI queue initialized\n");
 }
 
 /*
@@ -782,9 +751,7 @@
     if(ioapic == nil)
         return;
     
-    if(debug) 
-        fprint(2, "ioapic_eoi(%d): pending before=%#x\n", vector, 
-            ioapic->irq_pending);
+    dprint("ioapic_eoi(%d): pending before=%#x\n", vector, ioapic->irq_pending);
     
     lock(&ioapic->lock);
     for(i = 0; i < 24; i++){
@@ -796,9 +763,7 @@
             int level_triggered = (redir >> 15) & 1;
             int masked = (redir >> 16) & 1;
             
-            if(debug)
-                fprint(2, "ioapic_eoi: IRQ %d redir=%#llx level_trig=%d irq_level=%#x masked=%d\n",
-                       i, redir, level_triggered, ioapic->irq_level, masked);
+            dprint("ioapic_eoi: IRQ %d redir=%#llx level_trig=%d irq_level=%#x masked=%d\n", i, redir, level_triggered, ioapic->irq_level, masked);
             
             /* If level-triggered and line still high, handle re-delivery */
             if(level_triggered && (ioapic->irq_level & (1 << i))){
@@ -814,9 +779,7 @@
                 }
             }
             
-            if(debug) 
-                fprint(2, "ioapic_eoi: cleared IRQ %d, pending now=%#x\n", 
-                       i, ioapic->irq_pending);
+            dprint("ioapic_eoi: cleared IRQ %d, pending now=%#x\n", i, ioapic->irq_pending);
             break;
         }
     }
@@ -829,37 +792,13 @@
 void 
 ioapic_init(){
  
-    int fd;
-    char buf[64];
-    int i;
-
-    /* Create shared segment */
-    fd = create("#g/vmx.ioapic", OREAD, DMDIR | 0777);
-    if(fd >= 0) close(fd);
-    
-    fd = open("#g/vmx.ioapic/ctl", OWRITE);
-    if(fd < 0) sysfatal("open vmx.ioapic/ctl: %r");
-	snprint(buf, sizeof buf, "va 0x300000000 0x1000 sticky");
-    if(write(fd, buf, strlen(buf)) < 0)
-        fprint(2, "write vmx.ioapic/ctl: %r");
-    close(fd);
-    
-    ioapic = segattach(0, "vmx.ioapic", nil, 0x1000);
-    if(ioapic == (void*)-1)
-        sysfatal("segattach vmx.ioapic: %r");
-    
+    ioapic = mkseg("ioapic", 0x300000000, 0x1000);
     memset(ioapic, 0, sizeof(IOApic));
-    
-    /*
-     * CRITICAL: Set I/O APIC ID to match what MP table declares!
-     * MP table sets I/O APIC ID = nvcpu (after the CPU APIC IDs)
-     * The ID register format is: bits 24-27 = APIC ID
-     */
 
     ioapic->id = nvcpu << 24;  /* ID in bits 24-27 */
  
 	
-	for(i = 0; i < 24; i++){
+	for(int i = 0; i < 24; i++){
 	    if(i >= 16 && i <= 19){
 	        /* PCI interrupt pins: level-triggered, unmasked */
 	        ioapic->redir[i] = (1 << 15) | (48 + (i - 16));
@@ -869,8 +808,7 @@
 	    }
 	}
 
-    fprint(2, "IOAPIC initialized: id=%#ux (APIC ID %d)\n", 
-           ioapic->id, ioapic->id >> 24);
+    dprint("IOAPIC initialized: id=%#ux (APIC ID %d)\n", ioapic->id, ioapic->id >> 24);
 
 }
 
@@ -888,12 +826,12 @@
 		switch(off){
 		case 0x00: /* IOREGSEL */
 			ioapic->reg_sel[curcpuid] = val;
-			if(debug) fprint(2, "IOAPIC: IOREGSEL = %#x\n", val);
+			dprint("IOAPIC: IOREGSEL = %#x\n", val);
 			break;
 
 		case 0x10: /* IOWIN */
 		    sel = ioapic->reg_sel[curcpuid];
-		    if(debug) fprint(2, "IOAPIC: IOWIN write reg=%#x val=%#x\n", sel, val);
+		    dprint("IOAPIC: IOWIN write reg=%#x val=%#x\n", sel, val);
 		
 		    switch(sel){
 		    case 0x00: /* ID */
@@ -920,9 +858,9 @@
 						if (debug)
 		                if(idx == 2){
 		                    if(val & 0x10000)
-		                        fprint(2, "IOAPIC: IRQ 2 MASKED! val=%#x\n", val);
+		                        dprint("IOAPIC: IRQ 2 MASKED! val=%#x\n", val);
 		                    else
-		                        fprint(2, "IOAPIC: IRQ 2 UNMASKED val=%#x vector=%d\n", val, val & 0xFF);
+		                        dprint("IOAPIC: IRQ 2 UNMASKED val=%#x vector=%d\n", val, val & 0xFF);
 		                }
 		                
 /* Check if unmasking with pending interrupt OR line high */
@@ -951,8 +889,7 @@
 	                unlock(&ioapic->lock);
 		            }
 		
-		            if(debug) fprint(2, "IOAPIC: redir[%d] %s = %#llx\n",
-		                   idx, (sel & 1) ? "hi" : "lo", ioapic->redir[idx]);
+		            dprint("IOAPIC: redir[%d] %s = %#llx\n", idx, (sel & 1) ? "hi" : "lo", ioapic->redir[idx]);
 		        }
 		        break;
 		    }
@@ -959,7 +896,7 @@
 		done_ioapic_write:
     		break;
 		default:
-			if(debug) fprint(2, "IOAPIC: write to unknown offset %#x\n", off);
+			dprint("IOAPIC: write to unknown offset %#x\n", off);
 		}
 	} else {
 		switch(off){
@@ -996,7 +933,7 @@
 			}
 		}
 
-		if(debug) fprint(2, "IOAPIC: read off=%#x val=%#x\n", off, val);
+		dprint("IOAPIC: read off=%#x val=%#x\n", off, val);
 		setmovdest(val);
 	}
 }
@@ -1081,21 +1018,24 @@
 
     int pid = fork();
  
-	if (debug)
-		fprint(2, "CPU0: tscoff = %llud\n", ioapic->tsc_base);
+	dprint("CPU0: tscoff = %llud\n", ioapic->tsc_base);
 
     if(pid < 0)
         sysfatal("fork failed: %r");
 
     if(pid > 0){
+		vmx_register_child(pid);
         nactivecpu += 1;
         return;
     }     
 
+//	vmx_cleanup_init();
+
+	atnotify(vmx_notehandler, 1);
+
     /* Child process */
     curcpuid = nactivecpu + 1;
-	if (debug)
-	    fprint(2, "CPU%d: child started (pid %d)\n", curcpuid, getpid());
+	dprint("CPU%d: child started (pid %d)\n", curcpuid, getpid());
 
     if(ipiqueue != nil)
         ipiqueue->pids[curcpuid] = getpid();
@@ -1108,8 +1048,7 @@
     for(r = mmap; r != nil; r = r->next){
         modregion(r);
 	 
-		fprint(2, "CPU%d: mapped regions, first region v=%p segname=%s\n", 
-       		curcpuid, r->v, r->segname);
+		dprint("CPU%d: mapped regions, first region v=%p segname=%s\n", curcpuid, r->v, r->segname);
 	}
 
     /* Clear register cache - must reload for new context */
@@ -1245,12 +1184,15 @@
     if(waitch == nil ||   notifch == nil)
         sysfatal("CPU%d: chancreate failed: %r", curcpuid);
 
-    fprint(2, "CPU%d: starting at %#x:0000 (linear %#x)\n", 
-           curcpuid, sipi << 8, sipi << 12);
+    dprint("CPU%d: starting at %#x:0000 (linear %#x)\n",  curcpuid, sipi << 8, sipi << 12);
  
     rset("tscoff", 0);
 	cached_tscoff = 0;
 
+	
+    virtio_start_workers();  /* Start workers for CPU0 */
+    
+
     runloop();
     
     /* Should never return */
@@ -1476,7 +1418,7 @@
 lapic_write_tpr(u32int val)
 {
 	lapic_tpr[curcpuid] = val & 0xFF;
-	if(debug) fprint(2, "CPU%d: TPR = %#x\n", curcpuid, val);
+	dprint("CPU%d: TPR = %#x\n", curcpuid, val);
 }
 
 static void
@@ -1500,7 +1442,7 @@
 		}
 	}
 
-	if(debug) fprint(2, "CPU%d: EOI vec=%d\n", curcpuid, vec);
+	dprint("CPU%d: EOI vec=%d\n", curcpuid, vec);
 
 	if(vec >= 0){
 		isr_bitmap[vec >> 5] &= ~(1 << (vec & 31));
@@ -1513,7 +1455,7 @@
 static void
 lapic_write_ldr(u32int val)
 {
-    if(debug) fprint(2, "CPU%d: LDR = %#x\n", curcpuid, val);
+    dprint("CPU%d: LDR = %#x\n", curcpuid, val);
     if(ioapic != nil)
         ioapic->ldr[curcpuid] = val & 0xFF000000;  /* Only bits 31:24 are writable */
 }
@@ -1521,7 +1463,7 @@
 static void
 lapic_write_dfr(u32int val)
 {
-	if(debug) fprint(2, "CPU%d: DFR = %#x\n", curcpuid, val);
+	dprint("CPU%d: DFR = %#x\n", curcpuid, val);
 }
 
 static void
@@ -1528,8 +1470,7 @@
 lapic_write_svr(u32int val)
 {
 	lapic_svr[curcpuid] = val;
-	if(debug) fprint(2, "CPU%d: SVR = %#x (LAPIC %s)\n",
-	       curcpuid, val, (val & 0x100) ? "enabled" : "disabled");
+	dprint("CPU%d: SVR = %#x (LAPIC %s)\n", curcpuid, val, (val & 0x100) ? "enabled" : "disabled");
 }
 
 /*
@@ -1558,9 +1499,7 @@
 	USED(trigger);
 
 	/* Always log ICR writes - critical for debugging SMP issues */
-	 	if (debug)
-		fprint(2, "CPU%d: ICR_LO = %#x (vec=%d del=%d dest=%d short=%d)\n",
-	       curcpuid, val, vec, delmode, dest, shorthand);
+	 dprint("CPU%d: ICR_LO = %#x (vec=%d del=%d dest=%d short=%d)\n", curcpuid, val, vec, delmode, dest, shorthand);
 
 	switch(shorthand){
 	case 0: /* No shorthand - use destination field */
@@ -1570,8 +1509,7 @@
 		case 0: /* Fixed */
 		case 1: /* Lowest priority */
 			if(vec == 0){
-				if (debug)
-				fprint(2, "CPU%d: ignoring IPI with vector 0\n", curcpuid);
+				dprint("CPU%d: ignoring IPI with vector 0\n", curcpuid);
 				break;
 			}
 			ipi_queue(target, vec);
@@ -1578,8 +1516,7 @@
 			break;
 
 		case 4: /* NMI */
-			if (debug)
-			fprint(2, "CPU%d: NMI to CPU%d\n", curcpuid, target);
+			dprint("CPU%d: NMI to CPU%d\n", curcpuid, target);
 			if(target < nvcpu)
 				ipi_queue(target, 2);
 			break;
@@ -1586,31 +1523,29 @@
 
 		case 5: /* INIT */
 			
-			fprint(2, "CPU%d: INIT to CPU%d (init_sent was %d)\n", 
-			       curcpuid, target, target < MAXVCPU ? init_sent[target] : -1);
+			dprint("CPU%d: INIT to CPU%d (init_sent was %d)\n", curcpuid, target, target < MAXVCPU ? init_sent[target] : -1);
 			if(target > 0 && target < nvcpu){
 				init_sent[target] = 1;
-				fprint(2, "CPU%d: marked init_sent[%d] = 1\n", curcpuid, target);
+				dprint("CPU%d: marked init_sent[%d] = 1\n", curcpuid, target);
 			}
 			break;
 
 		case 6: /* Startup IPI */
-			if (debug)
-			fprint(2, "CPU%d: SIPI to CPU%d at %#x (init_sent=%d)\n",
+			dprint("CPU%d: SIPI to CPU%d at %#x (init_sent=%d)\n",
 			       curcpuid, target, vec << 12, 
 			       target < MAXVCPU ? init_sent[target] : -1);
+
 			if(target > 0 && target < nvcpu && init_sent[target] == 1){
 				init_sent[target] = 2;
-				fprint(2, "CPU%d: forking CPU%d now\n", curcpuid, target);
+				dprint("CPU%d: forking CPU%d now\n", curcpuid, target);
 				forkcpu(vec);
-				/* Wake self - might HLT soon and need kick */
 			 
+			 
 				coherence();
 				
-				fprint(2, "CPU%d: forkcpu returned\n", curcpuid);
+				dprint("CPU%d: forkcpu returned\n", curcpuid);
 			} else {
-				if (debug)
-				fprint(2, "CPU%d: SIPI ignored (target=%d nvcpu=%d init_sent=%d)\n",
+				dprint("CPU%d: SIPI ignored (target=%d nvcpu=%d init_sent=%d)\n",
 				       curcpuid, target, nvcpu, 
 				       target < MAXVCPU ? init_sent[target] : -1);
 			}
@@ -1620,15 +1555,13 @@
 
 	case 1: /* Self */
 		if((delmode == 0 || delmode == 1) && vec != 0){
-			if (debug)
-			fprint(2, "CPU%d: self-IPI (shorthand=1) vector %d (DEFERRED)\n", curcpuid, vec);
+			dprint("CPU%d: self-IPI (shorthand=1) vector %d (DEFERRED)\n", curcpuid, vec);
 			ipi_queue(curcpuid, vec);
 		}
 		break;
 
 	case 2: /* All including self */
-		if (debug)
-		fprint(2, "CPU%d: IPI to ALL (including self) vec=%d del=%d\n", curcpuid, vec, delmode);
+		dprint("CPU%d: IPI to ALL (including self) vec=%d del=%d\n", curcpuid, vec, delmode);
 		if((delmode == 0 || delmode == 1) && vec != 0){
 			for(i = 0; i < nvcpu; i++){
 				ipi_queue(i, vec);
@@ -1635,13 +1568,12 @@
 			}
 		}
 		if(delmode == 5){
-			fprint(2, "CPU%d: INIT broadcast\n", curcpuid);
+			dprint("CPU%d: INIT broadcast\n", curcpuid);
 		}
 		break;
 
 	case 3: /* All excluding self */
-		if (debug)
-		fprint(2, "CPU%d: IPI to ALL (excluding self) vec=%d del=%d\n", curcpuid, vec, delmode);
+		dprint("CPU%d: IPI to ALL (excluding self) vec=%d del=%d\n", curcpuid, vec, delmode);
 		if((delmode == 0 || delmode == 1) && vec != 0){
 			for(i = 0; i < nvcpu; i++){
 				if(i != curcpuid)
@@ -1656,8 +1588,7 @@
 lapic_write_icr_hi(u32int val)
 {
 	icr_hi_saved = val;
-	if(debug) fprint(2, "CPU%d: ICR_HI = %#x (dest=%d)\n",
-	       curcpuid, val, val >> 24);
+	dprint("CPU%d: ICR_HI = %#x (dest=%d)\n", curcpuid, val, val >> 24);
 }
  
 
@@ -1683,7 +1614,7 @@
 {
 	int vec = val & 0xFF;
 	if(vec != 0){
-		if(debug) fprint(2, "CPU%d: SELF_IPI register vec=%d\n", curcpuid, vec);
+		dprint("CPU%d: SELF_IPI register vec=%d\n", curcpuid, vec);
 		ipi_queue(curcpuid, vec);
 	}
 }
@@ -1698,7 +1629,7 @@
 	int iswrite = (ei->qual & 2) != 0;
 	u32int val;
 
-	if(debug) fprint(2, "LAPIC: off=%#x %s\n", off, iswrite ? "write" : "read");
+	dprint("LAPIC: off=%#x %s\n", off, iswrite ? "write" : "read");
 
 	if(iswrite){
 		val = getmovval();
@@ -1716,7 +1647,7 @@
 		case LAPIC_TIMER_DCR: lapic_write_divide_config(val); break;
 		case LAPIC_SELF_IPI:  lapic_write_self_ipi(val); break;
 		default:
-			if(debug) fprint(2, "LAPIC: write to %#x ignored\n", off);
+			dprint("LAPIC: write to %#x ignored\n", off);
 		}
 	} else {
 		switch(off){
@@ -1751,7 +1682,7 @@
 
 		default:
 			val = 0;
-			if(debug) fprint(2, "LAPIC: read from %#x returning 0\n", off);
+			dprint("LAPIC: read from %#x returning 0\n", off);
 		}
 
 		setmovdest(val);
@@ -1793,8 +1724,7 @@
 	eptfault_count++;
 	
 	/* Always log EPT faults to help debug SMP issues */
-	if (debug)
-	fprint(2, "CPU%d EPT[%lld]: pa=%#llux va=%#llux qual=%#llux\n",
+	dprint("CPU%d EPT[%lld]: pa=%#llux va=%#llux qual=%#llux\n",
 	       curcpuid, eptfault_count, ei->pa, ei->va, ei->qual);
 
 	/* I/O APIC MMIO */
@@ -1818,10 +1748,10 @@
 	/* Other EPT faults - log and continue */
 	if(ei->pa < 0x1000){
 		u32int val = getmovval();
-		fprint(2, "CPU%d LOW MEM WRITE: pa=%#llx val=%#x\n", curcpuid, ei->pa, val);
+		dprint("CPU%d LOW MEM WRITE: pa=%#llx val=%#x\n", curcpuid, ei->pa, val);
 	} else {
 		/* Unexpected EPT fault - could indicate memory mapping issue */
-		fprint(2, "CPU%d UNEXPECTED EPT: pa=%#llx va=%#llx qual=%#llx\n",
+		dprint("CPU%d UNEXPECTED EPT: pa=%#llx va=%#llx qual=%#llx\n",
 		       curcpuid, ei->pa, ei->va, ei->qual);
 	}
 
@@ -2147,7 +2077,7 @@
 
 	ax = rget(RAX);
 	cx = rget(RCX);
-	if(debug) fprint(2, "CPU%d: CPUID eax=%#x ecx=%#x\n", curcpuid, ax, cx);
+	dprint("CPU%d: CPUID eax=%#x ecx=%#x\n", curcpuid, ax, cx);
 	bx = dx = 0;
 	cp = getcpuid(ax, cx);
 
@@ -2272,7 +2202,7 @@
     bx = 0;
     cx = 0;
     dx = (1 << 8);  /* Invariant TSC */
-    fprint(2, "CPUID[80000007]: returning dx=%#x (InvariantTSC=%d)\n", dx, (dx >> 8) & 1);
+    dprint("CPUID[80000007]: returning dx=%#x (InvariantTSC=%d)\n", dx, (dx >> 8) & 1);
     break;
 	case 0x80000008: goto literal;
 	literal:
@@ -2307,7 +2237,7 @@
 	cx = rget(RCX);
 	val = (uvlong)rget(RDX) << 32 | rget(RAX);
 
-	if(debug) fprint(2, "CPU%d: MSR %s cx=%#x\n", curcpuid, rd?"read":"write", cx);
+	dprint("CPU%d: MSR %s cx=%#x\n", curcpuid, rd?"read":"write", cx);
 
 	switch(cx){
 	case 0x10: /* IA32_TIME_STAMP_COUNTER */
@@ -2347,8 +2277,7 @@
     	if(rd)
         	val = lapic_read_tsc_deadline();
     	else {
-			if (debug)
-				fprint(2, "CPU%d: TSC_DEADLINE=%llud\n", curcpuid, val);
+			dprint("CPU%d: TSC_DEADLINE=%llud\n", curcpuid, val);
         	lapic_write_tsc_deadline(val);
 			 
 			pvclock_update(curcpuid);  /* Keep kvmclock in sync */
@@ -2453,7 +2382,7 @@
 	exit_count++;
 	vlong now = nanosec();
 	if(now - last_report > 1000000000LL) {  // Every 1 second
-    	fprint(2, "EXIT RATE: %llud/sec\n", exit_count);
+    	dprint("EXIT RATE: %llud/sec\n", exit_count);
     	exit_count = 0;
     	last_report = now;
 	}
@@ -2460,16 +2389,15 @@
 
 	static vlong lastpreempt;
       
+	dprint("EXIT: %s\n", msg);
     
-	if (debug) {
-		fprint(2, "EXIT: %s\n", msg);
-    
     if(strncmp(msg, "preempt", 7) == 0){
         if(lastpreempt != 0)
-            fprint(2, "PREEMPT CPU%d: delta=%lldms\n", curcpuid, (now - lastpreempt)/1000000);
+            dprint("PREEMPT CPU%d: delta=%lldms\n", curcpuid, (now - lastpreempt)/1000000);
         lastpreempt = now;
     }
-	}
+	
+
      
     strcpy(msgc, msg);
     nf = tokenize(msgc, f, nelem(f));
--- a/fns.h
+++ b/fns.h
@@ -1,8 +1,12 @@
 #define MIN(a,b) ((a)<(b)?(a):(b))
 #define MAX(a,b) ((a)>(b)?(a):(b))
-
+void virtio_start_workers(void);
 int irr_pending(void);
-
+void *mkseg(char *name, uintptr va, ulong size);
+void rmseg(char *name);
+extern int vmx_notehandler(void *, char *);
+extern void vmx_cleanup_init(void);
+extern void vmx_register_child(int pid);
 void lapic_timer_init(void);
 void ipi_send(int, int);
 u32int lapic_divide_value(u32int);
--- a/hpet.c
+++ b/hpet.c
@@ -170,15 +170,10 @@
 	/* Fire interrupts outside lock */
 	for(i = 0; i < 3; i++){
 		if(should_fire[i]){
-			if(debug)
-				fprint(2, "HPET: timer %d fired! irq=%d periodic=%d\n", 
+			dprint("HPET: timer %d fired! irq=%d periodic=%d\n", 
 				       i, irqs[i], is_periodic[i]);
 			
 			ioapic_irqline_smp(irqs[i], 1);
-//			ioapic_set_irq(irqs[i], 1);
-			
-//			if(state == VMHALT)
-//				state = VMRUNNING;
 		}
 	}
 }
@@ -223,8 +218,7 @@
 	int timer;
 	int state_changed = 0;  /* Track if we need to kick CPU0 */
 	
-	if(debug)
-		fprint(2, "HPET write: offset=%#x val=%#x size=%d\n", offset, val, size);
+	dprint("HPET write: offset=%#x val=%#x size=%d\n", offset, val, size);
 	
 	lock(&hpet->lock);
 	
@@ -242,8 +236,7 @@
 		if(was_enabled && !new_enabled){
 			/* Stopping: save current counter value */
 			hpet->counter_offset = hpet_counter_unlocked();
-			if(debug)
-				fprint(2, "HPET: DISABLED counter=%#llx\n", hpet->counter_offset);
+			dprint("HPET: DISABLED counter=%#llx\n", hpet->counter_offset);
 		}
 		
 		if(size == 4)
@@ -256,9 +249,8 @@
 		if(!was_enabled && new_enabled){
 			/* Starting: record real host time */
 			hpet->time_at_enable = nanosec();
-			if(debug)
-				fprint(2, "HPET: ENABLED time=%llud counter_offset=%#llx legacy=%d\n", 
-				       hpet->time_at_enable, hpet->counter_offset, (hpet->cfg >> 1) & 1);
+			dprint("HPET: ENABLED time=%llud counter_offset=%#llx legacy=%d\n", 
+				hpet->time_at_enable, hpet->counter_offset, (hpet->cfg >> 1) & 1);
 		}
 		
 		state_changed = 1;
@@ -294,8 +286,7 @@
 				hpet->counter_offset = (hpet->counter_offset & 0xFFFFFFFF00000000ULL) | val;
 			else
 				hpet->counter_offset = val;
-			if(debug)
-				fprint(2, "HPET: counter set to %#llx\n", hpet->counter_offset);
+			dprint("HPET: counter set to %#llx\n", hpet->counter_offset);
 		}
 		break;
 		
@@ -302,8 +293,7 @@
 	case HPET_COUNTER + 4:
 		if(!hpet->enabled){
 			hpet->counter_offset = (hpet->counter_offset & 0xFFFFFFFF) | ((u64int)val << 32);
-			if(debug)
-				fprint(2, "HPET: counter set to %#llx\n", hpet->counter_offset);
+			dprint("HPET: counter set to %#llx\n", hpet->counter_offset);
 		}
 		break;
 	
@@ -311,8 +301,7 @@
 	case HPET_T0_CFG:
 		hpet->timer[0].cfg = (size == 4) ? 
 			(hpet->timer[0].cfg & 0xFFFFFFFF00000000ULL) | val : val;
-		if(debug)
-			fprint(2, "HPET: T0 cfg=%#llx int_enb=%d periodic=%d\n", 
+		dprint("HPET: T0 cfg=%#llx int_enb=%d periodic=%d\n", 
 			       hpet->timer[0].cfg, 
 			       (hpet->timer[0].cfg >> 2) & 1,
 			       (hpet->timer[0].cfg >> 3) & 1);
@@ -329,12 +318,10 @@
 		/* If VAL_SET bit is set in periodic mode, this also sets the period */
 		if((hpet->timer[0].cfg & TN_TYPE) && (hpet->timer[0].cfg & TN_VAL_SET)){
 			hpet->timer[0].period = hpet->timer[0].cmp;
-			if(debug)
-				fprint(2, "HPET: T0 period set to %#llx\n", hpet->timer[0].period);
+			dprint("HPET: T0 period set to %#llx\n", hpet->timer[0].period);
 		}
 		hpet->timer[0].active = 1;
-		if(debug)
-			fprint(2, "HPET: T0 cmp=%#llx counter=%#llx active=1 int_enb=%d\n", 
+		dprint("HPET: T0 cmp=%#llx counter=%#llx active=1 int_enb=%d\n", 
 			       hpet->timer[0].cmp, hpet_counter_unlocked(),
 			       (hpet->timer[0].cfg & TN_INT_ENB) ? 1 : 0);
 		state_changed = 1;
@@ -416,5 +403,5 @@
 	hpet->counter_offset = 0;
 	hpet->time_at_enable = 0;
 	
-	fprint(2, "HPET: init (shared via vioalloc)\n");
+	dprint("HPET: init (shared via vioalloc)\n");
 }
--- a/io.c
+++ b/io.c
@@ -267,13 +267,11 @@
 			continue;
 		
 		lock(&kbdshared->lk);
-		if (debug)
-		fprint(2, "IOBUF: kbdreader WRITE r=%d w=%d char=%#x\n",
+		dprint("IOBUF: kbdreader WRITE r=%d w=%d char=%#x\n",
 		    (int)kbdshared->r, (int)kbdshared->w, c);
 		kbdshared->buf[kbdshared->w & (KBD_BUF_SIZE - 1)] = c;
 		kbdshared->w += 1;
-		if (debug)
-		fprint(2, "IOBUF: kbdreader AFTER r=%d w=%d\n", (int)kbdshared->r, (int)kbdshared->w);
+		dprint("IOBUF: kbdreader AFTER r=%d w=%d\n", (int)kbdshared->r, (int)kbdshared->w);
 		unlock(&kbdshared->lk);
 		
 		sendnotif(i8042kick, nil);
@@ -688,8 +686,7 @@
 	mouse->state = MOUSESTREAM | MOUSEREP;
 	mouse->res = 2;
 	mouse->rate = 100;
-	if (debug)
-	fprint(2, "MOUSE: mouseps2init done, mouse=%p\n", mouse);
+	dprint("MOUSE: mouseps2init done, mouse=%p\n", mouse);
 }
 
 static void
@@ -878,8 +875,7 @@
 mousecmd(u8int val)
 {
 	/* incmd is already set and buffer is flushed by caller (i8042io 0xd4 case) */
-	if (debug)
-		fprint(2, "MOUSE: cmd %#x state=%#x bufr=%d bufw=%d\n", 
+	dprint("MOUSE: cmd %#x state=%#x bufr=%d bufw=%d\n", 
            val, mouse->state, mouse->bufr, mouse->bufw);
 
 	if((mouse->state & MOUSEWRAP) != 0 && val != 0xec && val != 0xff){
@@ -931,8 +927,7 @@
 		case 0xf2: 
 			mousecmdputc(0xfa); 
 			mousecmdputc(mouse->id); 
-			if (debug)
-				fprint(2, "MOUSE: F2 sent ACK+ID - cmdbuf[0]=%#x cmdbuf[1]=%#x cmdr=%d cmdw=%d\n",
+			dprint("MOUSE: F2 sent ACK+ID - cmdbuf[0]=%#x cmdbuf[1]=%#x cmdr=%d cmdw=%d\n",
            mouse->cmdbuf[0], mouse->cmdbuf[1], mouse->cmdr, mouse->cmdw);
     
 			clearmouse(); break; /* report device id */
@@ -965,8 +960,7 @@
 	lock(&i8042->lk);
 	if(mouse->cmdr != mouse->cmdw && i8042->buf == 0){
 		u8int c = mouse->cmdbuf[mouse->cmdr++ & 7];
-		if (debug)
-			fprint(2, "MOUSE: cmd delivering response 0x%x\n", c);
+		dprint("MOUSE: cmd delivering response 0x%x\n", c);
 		i8042->buf = 0x200 | c;
 		i8042->stat |= 0x21;
 		i8042->oport |= 0x20;
@@ -1015,7 +1009,7 @@
 	if(i8042->buf != 0){
 		bufstuck++;
 		if(bufstuck == 10000)
-			fprint(2, "STUCK: buf=%#x stuck for 10000 kicks, cfg=%#x stat=%#x\n",
+			dprint("STUCK: buf=%#x stuck for 10000 kicks, cfg=%#x stat=%#x\n",
 				i8042->buf, i8042->cfg, i8042->stat);
 	} else {
 		bufstuck = 0;
@@ -1022,8 +1016,7 @@
 	}
 	
 	if((kickcount % 5000) == 0){
-		if (debug)
-		fprint(2, "KICK[%d]: mouse=%p buf=%#x cfg=%#x incmd=%d bufr=%d bufw=%d cmdr=%d cmdw=%d irq12=%d\n",
+		dprint("KICK[%d]: mouse=%p buf=%#x cfg=%#x incmd=%d bufr=%d bufw=%d cmdr=%d cmdw=%d irq12=%d\n",
 			kickcount, mouse, i8042->buf, i8042->cfg,
 			mouse ? mouse->incmd : -1,
 			mouse ? mouse->bufr : -1,
@@ -1116,8 +1109,7 @@
 	if(i8042->cmd == 0){
 		i8042->mouseactive = 1;
 		i8042->cmd = -1;
-		if (debug)
-		fprint(2, "MOUSE: init mouseactive=1\n");
+		dprint("MOUSE: init mouseactive=1\n");
 	}
 	unlock(&i8042->lk);
 
@@ -1124,8 +1116,7 @@
 	val = (u8int)val;
 	switch(isin << 16 | port){
 	case 0x60:
-		if (debug)
-		fprint(2, "I8042: port 0x60 WRITE val=%#x cmd=%d\n", val, i8042->cmd);
+		dprint("I8042: port 0x60 WRITE val=%#x cmd=%d\n", val, i8042->cmd);
 		lock(&i8042->lk);
 		i8042->stat &= ~8;
 		switch(i8042->cmd){
@@ -1134,8 +1125,7 @@
 			/* Don't let config byte disable mouse - only A7/A8 commands do that.
 			 * Guests without psmouse driver write 0x65 (bit 5 set) at end of probe,
 			 * which would disable our mouse even though we want it active. */
-			if (debug)
-			fprint(2, "MOUSE: cfg=%#x mouseactive=%d (unchanged)\n", val, i8042->mouseactive);
+			dprint("MOUSE: cfg=%#x mouseactive=%d (unchanged)\n", val, i8042->mouseactive);
 			break;
 		case 0xd1:
 			i8042->oport = val;
@@ -1156,8 +1146,7 @@
 			lock(&i8042->lk);
 			break;
 case 0xd4:
-	if (debug)
-	    fprint(2, "MOUSE: forwarding %#x to mouse, cfg=%#x buf=%#x\n", val, i8042->cfg, i8042->buf);
+	dprint("MOUSE: forwarding %#x to mouse, cfg=%#x buf=%#x\n", val, i8042->cfg, i8042->buf);
     /* Set incmd BEFORE unlocking to prevent race with i8042kick */
     mouse->incmd = 1;
     /* Clear any unread mouse data from i8042->buf and flush cmdbuf */
@@ -1171,19 +1160,16 @@
     /* mousecmd may have already delivered first byte */
     /* Only deliver here if buffer is still empty */
     lock(&i8042->lk);
-	if (debug)
-	    fprint(2, "MOUSE: after mousecmd, buf=%#x cmdr=%d cmdw=%d\n", i8042->buf, mouse->cmdr, mouse->cmdw);
+	dprint("MOUSE: after mousecmd, buf=%#x cmdr=%d cmdw=%d\n", i8042->buf, mouse->cmdr, mouse->cmdw);
     if(i8042->buf == 0 && mouse->cmdr != mouse->cmdw){
         uchar c = mouse->cmdbuf[mouse->cmdr++ & 7];
-		if (debug)
-	        fprint(2, "MOUSE: D4 delivering response %#x\n", c);
+		dprint("MOUSE: D4 delivering response %#x\n", c);
         i8042->buf = 0x200 | c;
         i8042->stat |= 0x21;  /* OBF (bit 0) + aux data flag (bit 5) */
         i8042->oport |= 0x20;
         if((i8042->cfg & 2) != 0){
             unlock(&i8042->lk);
-			if (debug)
-	            fprint(2, "MOUSE: firing IRQ 12 for response %#x\n", c);
+			dprint("MOUSE: firing IRQ 12 for response %#x\n", c);
             ioapic_irqline_smp(12, 1);
             lock(&i8042->lk);
         }
@@ -1203,8 +1189,7 @@
 	case 0x10060:
 		i8042kick(nil);
 		lock(&i8042->lk);
-		if (debug)
-		fprint(2, "I8042: i8042io READ buf=%#x stat=%#x\n", i8042->buf, i8042->stat);
+		dprint("I8042: i8042io READ buf=%#x stat=%#x\n", i8042->buf, i8042->stat);
 		rc = i8042->buf;
 		unlock(&i8042->lk);
 		i8042putbuf(0);
@@ -1211,8 +1196,7 @@
 		return rc;
 		
 	case 0x64:
-		if (debug)
-		fprint(2, "I8042: port 0x64 WRITE cmd=%#x\n", val);
+		dprint("I8042: port 0x64 WRITE cmd=%#x\n", val);
 		lock(&i8042->lk);
 		i8042->stat |= 8;
 		switch(val){
@@ -1236,8 +1220,7 @@
 			unlock(&i8042->lk);
 			return 0;
 		case 0xa9:
-			if (debug)
-			fprint(2, "I8042: test aux port (0xA9)\n");
+			dprint("I8042: test aux port (0xA9)\n");
 			unlock(&i8042->lk);
 			i8042putbuf(0x400); // was 0x400 ?
 			return 0;
@@ -1263,8 +1246,7 @@
 			i8042putbuf(0x400 | oport);
 			return 0;
 		case 0x60: case 0xd1: case 0xd2: case 0xd3: case 0xd4:
-			if (debug)
-			fprint(2, "I8042: aux write cmd=0xD4, data will follow\n");
+			dprint("I8042: aux write cmd=0xD4, data will follow\n");
 			i8042->cmd = val;
 			unlock(&i8042->lk);
 			return 0;
@@ -1285,8 +1267,7 @@
 		i8042kick(nil);
 		lock(&i8042->lk);
 		rc = i8042->stat | i8042->cfg & 4;
-		if (debug)
-		fprint(2, "I8042: port 0x64 READ stat=%#x\n", rc);
+		dprint("I8042: port 0x64 READ stat=%#x\n", rc);
 		unlock(&i8042->lk);
 		return rc;
 	}
@@ -1657,7 +1638,7 @@
         proccreate(uarttxproc, (void*)(uintptr)n, 4096);
     }
     
-    fprint(2, "UART%d: initialized infd=%d outfd=%d\n", n, uart[n].infd, uart[n].outfd);
+    dprint("UART%d: initialized infd=%d outfd=%d\n", n, uart[n].infd, uart[n].outfd);
 }
 
 
--- a/io_timer.c
+++ b/io_timer.c
@@ -272,7 +272,7 @@
 	pitshared = vioalloc(sizeof(PitShared));
 	memset(pitshared, 0, sizeof(PitShared));
 	pitshared->ch[0].state = 1;
-	fprint(2, "PIT: initialized (shared)\n");
+	dprint("PIT: initialized (shared)\n");
 }
 
 /* ============================================================
@@ -489,5 +489,5 @@
 	rtcshared->rtcnext = -1;
 	rtcshared->cmosinit = 0;
 	
-	fprint(2, "RTC: initialized (shared)\n");
+	dprint("RTC: initialized (shared)\n");
 }
--- a/mkfile
+++ b/mkfile
@@ -22,6 +22,7 @@
 	hpet.$O \
 	pvclock.$O \
 	io_timer.$O \
+	cleanup.$O \
 
 </sys/src/cmd/mkone
 
--- a/mptable.c
+++ b/mptable.c
@@ -298,7 +298,7 @@
 	cfg[7] = -mpchecksum(cfg, tbllen);
 	
 	/* Debug: print what we created */
-	vmerror("mpmktable: created MP table at 0xF0000");
-	vmerror("  %d CPUs, %d entries, %d bytes", ncpu, entries, tbllen);
-	vmerror("  PCI bus ID=%d, ISA bus ID=%d", BUS_PCI, BUS_ISA);
+	dprint("mpmktable: created MP table at 0xF0000");
+	dprint("  %d CPUs, %d entries, %d bytes", ncpu, entries, tbllen);
+	dprint("  PCI bus ID=%d, ISA bus ID=%d", BUS_PCI, BUS_ISA);
 }
--- a/pci.c
+++ b/pci.c
@@ -7,6 +7,9 @@
 PCIDev *pcidevs;
 PCIBar membars, iobars;
 
+
+static PCIDev *pcidevtab[32];
+
 PciShared *pcishared;
 
 static int
@@ -65,6 +68,10 @@
 	d->capalloc = 64;
 	pcidevs = d;
 
+	int dev = 0;
+	dev = (bdf >> 11) & 0x1f;
+    pcidevtab[dev] = d;
+
 	pciregister(d);
 
 	return d;  
@@ -76,7 +83,7 @@
 {
     static int dev = 1;
     u32int bdf = BDF(0, dev, 0);
-    fprint(2, "allocbdf: dev=%d BDF=%#x\n", dev, bdf);
+    dprint("allocbdf: dev=%d BDF=%#x\n", dev, bdf);
     dev++;
     return bdf;
 }
@@ -201,22 +208,20 @@
 	*p = c;
 	return c;
 }
+ 
 
 static PCIDev *
 findpcidev(u32int bdf)
 {
-    PCIDev *d;
-	if (debug)
-    fprint(2, "findpcidev: looking for bdf=%#x\n", bdf);
-    for(d = pcidevs; d != nil; d = d->next) {
-		if (debug)
-        fprint(2, "  checking d->bdf=%#x\n", d->bdf);
-        if(d->bdf == bdf)
-            return d;
-    }
-	if (debug)
-    fprint(2, "  not found!\n");
-    return nil;
+    int dev;
+    
+    if((bdf & 0xff0000) != 0)  /* bus != 0 */
+        return nil;
+    if((bdf & 0x700) != 0)     /* function != 0 */
+        return nil;
+    
+    dev = (bdf >> 11) & 0x1f;
+    return pcidevtab[dev];
 }
 
 static PCICap *
@@ -238,7 +243,7 @@
     int n, idx;
 
     /* Lock and sync from shared memory */
-    if(pcishared != nil){
+    if(pcishared != nil && d->cache_gen != pcishared->generation){
         lock(&pcishared->lock);
         idx = d->sharedidx; 
         if(idx >= 0){
@@ -247,15 +252,14 @@
             for(n = 0; n < 6; n++)
                 d->bar[n].addr = pcishared->dev[idx].bar_addr[n];
         }
+		d->cache_gen = pcishared->generation;
         unlock(&pcishared->lock);
     }
-	if (debug)
-	fprint(2, "pciread: d=%p bdf=%#x addr=%#x\n", d, d->bdf, addr);
+	dprint("pciread: d=%p bdf=%#x addr=%#x\n", d, d->bdf, addr);
 
 	switch(addr){
 	case 0x00: 
-		if (debug)
-		fprint(2, "pciread: returning viddid=%#x\n", d->viddid);
+		dprint("pciread: returning viddid=%#x\n", d->viddid);
 		return d->viddid;
 	case 0x04: return 0xa00000 | (d->cap != nil ? 1<<20 : 0) | d->ctrl;
 	case 0x08: return d->clrev;
@@ -292,7 +296,7 @@
     int n, idx;
 
     /* Lock and sync from shared memory */
-    if(pcishared != nil){
+    if(pcishared != nil && d->cache_gen != pcishared->generation){
         //lock(&pcishared->lock);
         idx = d->sharedidx;
         if(idx >= 0){
@@ -301,15 +305,14 @@
             for(n = 0; n < 6; n++)
                 d->bar[n].addr = pcishared->dev[idx].bar_addr[n];
         }
+		d->cache_gen = pcishared->generation;
         //unlock(&pcishared->lock);
     }
-	if (debug)
-	fprint(2, "pciread: d=%p bdf=%#x addr=%#x\n", d, d->bdf, addr);
+	dprint("pciread: d=%p bdf=%#x addr=%#x\n", d, d->bdf, addr);
 
 	switch(addr){
 	case 0x00: 
-		if (debug)
-		fprint(2, "pciread: returning viddid=%#x\n", d->viddid);
+		dprint("pciread: returning viddid=%#x\n", d->viddid);
 		return d->viddid;
 	case 0x04: return 0xa00000 | (d->cap != nil ? 1<<20 : 0) | d->ctrl;
 	case 0x08: return d->clrev;
@@ -390,6 +393,7 @@
             pcishared->dev[idx].irqno = d->irqno;
             for(n = 0; n < 6; n++)
                 pcishared->dev[idx].bar_addr[n] = d->bar[n].addr;
+			pcishared->generation++; 
         }
         unlock(&pcishared->lock);
     }
@@ -527,23 +531,19 @@
     d->irqactive = status != 0;
     devno = (d->bdf >> 11) & 0x1f;
     pin = 16 + (devno % 4);
-    if (debug)
-    fprint(2, "pciirq: bdf=%#x devno=%d irqno=%d status=%d pin=%d ioapic=%p\n",
+    dprint("pciirq: bdf=%#x devno=%d irqno=%d status=%d pin=%d ioapic=%p\n",
            d->bdf, devno, d->irqno, status, pin, ioapic);
     
     if(ioapic != nil){
-		if (debug)
-        fprint(2, "pciirq: using IOAPIC path, calling ioapic_set_irq(%d, %d)\n", 
+		dprint("pciirq: using IOAPIC path, calling ioapic_set_irq(%d, %d)\n", 
                pin, d->irqactive);
 		ioapic_irqline_smp(pin, d->irqactive); 
     } else if(d->irqno < 16){
-		if (debug)
-        fprint(2, "pciirq: using legacy PIC path, irqline(%d, %d)\n",
+		dprint("pciirq: using legacy PIC path, irqline(%d, %d)\n",
                d->irqno, d->irqactive ? 0 : 1);
         ioapic_irqline_smp(d->irqno, d->irqactive ? 0 : 1);
     } else {
-		if (debug)
-        fprint(2, "pciirq: NO PATH TAKEN - ioapic=%p irqno=%d\n", 
+		dprint("pciirq: NO PATH TAKEN - ioapic=%p irqno=%d\n", 
                ioapic, d->irqno);
     }
 }
@@ -627,6 +627,7 @@
                     pcishared->dev[idx].bar_addr[i] = d->bar[i].addr;
             }
         }
+		pcishared->generation++;
         unlock(&pcishared->lock);
     }
 
@@ -641,21 +642,21 @@
 	extern IOApic *ioapic;
 	extern u32int lapic_svr[];
 	
-	fprint(2, "=== PCI Dump ===\n");
-	fprint(2, "ioapic=%p, lapic_svr[0]=%#ux (APIC %s)\n", 
+	dprint("=== PCI Dump ===\n");
+	dprint("ioapic=%p, lapic_svr[0]=%#ux (APIC %s)\n", 
 		ioapic, lapic_svr[0], (lapic_svr[0] & 0x100) ? "enabled" : "disabled");
 	
 	for(d = pcidevs, i = 0; d != nil; d = d->next, i++){
-		fprint(2, "[%d] bdf=%#ux viddid=%#ux class=%#ux irqno=%d irqactive=%d ctrl=%#ux\n",
+		dprint("[%d] bdf=%#ux viddid=%#ux class=%#ux irqno=%d irqactive=%d ctrl=%#ux\n",
 			i, d->bdf, d->viddid, d->clrev >> 8, d->irqno, d->irqactive, d->ctrl);
 		for(j = 0; j < 6; j++){
 			b = &d->bar[j];
 			if(b->length > 0)
-				fprint(2, "    BAR%d: type=%#ux addr=%#ux len=%#ux io=%p\n",
+				dprint("    BAR%d: type=%#ux addr=%#ux len=%#ux io=%p\n",
 					j, b->type, b->addr, b->length, b->io);
 		}
 	}
-	fprint(2, "================\n");
+	dprint("================\n");
 }
 
 PciShared *pcishared;
@@ -663,30 +664,7 @@
 void
 pcisharedinit(void)
 {
-    int fd;
-    char buf[128];
-    
-	remove("#g/vmx.pci/ctl");
-	remove("#g/vmx.pci");
-
-    snprint(buf, sizeof(buf), "#g/vmx.pci");
-    fd = create(buf, OREAD, DMDIR | 0777);
-    if(fd >= 0) close(fd);
-    
-    snprint(buf, sizeof(buf), "#g/vmx.pci/ctl");
-    fd = open(buf, OWRITE|OTRUNC);
-    if(fd < 0) {
-        fprint(2, "pcisharedinit: cannot open ctl: %r\n");
-        return;
-    }
-    snprint(buf, sizeof(buf), "va 0x300002000 0x1000 sticky");
-    write(fd, buf, strlen(buf));
-    close(fd);
-    
-    pcishared = segattach(0, "vmx.pci", nil, 0x1000);
-    if(pcishared == (void*)-1)
-        sysfatal("segattach vmx.pci: %r");
-    
+    pcishared = mkseg("pci", 0x300002000, 0x1000);
     memset(pcishared, 0, sizeof(PciShared));
 }
 
--- a/pvclock.c
+++ b/pvclock.c
@@ -78,7 +78,7 @@
 	
 	tsc_mul = (u32int)mul64;
 	
-	fprint(2, "pvclock: tsc_freq=%llud mul=%ud shift=%d\n",
+	dprint("pvclock: tsc_freq=%llud mul=%ud shift=%d\n",
 	       tsc_freq, tsc_mul, tsc_shift);
 }
 
@@ -87,7 +87,7 @@
 {
 	compute_tsc_params();
 	boot_ns = nanosec();
-	fprint(2, "pvclock: boot_ns=%llud\n", boot_ns);
+	dprint("pvclock: boot_ns=%llud\n", boot_ns);
 }
 
 /*
@@ -137,7 +137,7 @@
 static void
 pvclock_setup(int cpu)
 {
-	fprint(2, "CPU%d: pvclock setup gpa=%#llux tscoff=%lld\n",
+	dprint("CPU%d: pvclock setup gpa=%#llux tscoff=%lld\n",
 	       cpu, pvclock_gpa[cpu], (vlong)cached_tscoff);
 	pvclock_update(cpu);
 }
@@ -169,7 +169,7 @@
 	coherence();
 	wc->version = 2;
 	
-	fprint(2, "wallclock: sec=%lld at boot\n", wall_sec);
+	dprint("wallclock: sec=%lld at boot\n", wall_sec);
 }
 
 int
@@ -184,7 +184,7 @@
 			pvclock_gpa[cpu] = *val & ~1ULL;
 			pvclock_enabled[cpu] = *val & 1;
 			
-			fprint(2, "CPU%d: pvclock MSR write gpa=%#llux enabled=%d\n",
+			dprint("CPU%d: pvclock MSR write gpa=%#llux enabled=%d\n",
 			       cpu, pvclock_gpa[cpu], pvclock_enabled[cpu]);
 			
 			if(pvclock_enabled[cpu])
@@ -197,7 +197,7 @@
 	case MSR_KVM_WALL_CLOCK_NEW:
 		if(write){
 			wallclock_gpa = *val;
-			fprint(2, "wallclock: GPA set to %#llux\n", wallclock_gpa);
+			dprint("wallclock: GPA set to %#llux\n", wallclock_gpa);
 			wallclock_update();
 		} else {
 			*val = wallclock_gpa;
--- a/timer.c
+++ b/timer.c
@@ -332,8 +332,7 @@
 	
 	timer_initialized = 1;
 	
-	fprint(2, "CPU%d: timer_init tsc_freq=%llud preempt_shift=%d backstop=%lldms\n", 
-		curcpuid, tsc_freq, preempt_shift, BACKSTOP_NS_FIXED / 1000000);
+	dprint("CPU%d: timer_init tsc_freq=%llud preempt_shift=%d backstop=%lldms\n", curcpuid, tsc_freq, preempt_shift, BACKSTOP_NS_FIXED / 1000000);
 }
 
 /*
@@ -459,9 +458,7 @@
 	la->timer_active = 1;
 	la->timer_current = initial;
 	
-	if(debug)
-		fprint(2, "LAPIC%d: timer started, initial=%ud divide=%ud period=%lldns deadline=%lld\n",
-			curcpuid, initial, divide, period_ns, la->timer_deadline);
+	dprint("LAPIC%d: timer started, initial=%ud divide=%ud period=%lldns deadline=%lld\n", curcpuid, initial, divide, period_ns, la->timer_deadline);
 	
 	/* Only arm if not masked */
 	if(!(la->lvt_timer & (1<<16)))
@@ -489,9 +486,7 @@
 	vector = la->lvt_timer & 0xFF;
 	periodic = (la->lvt_timer >> 17) & 1;
 	
-	if(debug)
-		fprint(2, "LAPIC%d: FIRING timer! now=%lld deadline=%lld vec=%d\n",
-			curcpuid, now, la->timer_deadline, vector);
+	dprint("LAPIC%d: FIRING timer! now=%lld deadline=%lld vec=%d\n", curcpuid, now, la->timer_deadline, vector);
 	
 	/* Fire the interrupt if not masked and vector is valid */
 	if(vector != 0 && (la->lvt_timer & (1<<16)) == 0)
@@ -534,8 +529,7 @@
 {
 	LApic *la = &lapic;
 	
-	if(debug)
-		fprint(2, "LAPIC%d: TIMER_INIT = %ud\n", curcpuid, val);
+	dprint("LAPIC%d: TIMER_INIT = %ud\n", curcpuid, val);
 	
 	la->timer_initial = val;
 	lapic_timer_start();
@@ -563,9 +557,7 @@
 		timer_cancel(TIMER_DEADLINE);
 	}
 	
-	if(debug)
-		fprint(2, "LAPIC%d: LVT_TIMER = %#x (vec=%d mode=%d mask=%d)\n",
-			curcpuid, val, val & 0xFF, new_mode, (val >> 16) & 1);
+	dprint("LAPIC%d: LVT_TIMER = %#x (vec=%d mode=%d mask=%d)\n", curcpuid, val, val & 0xFF, new_mode, (val >> 16) & 1);
 	
 	/* Only start counter-based timer if not in deadline mode */
 	if(new_mode != MODE_DEADLINE && !(val & (1<<16)) && la->timer_initial > 0)
@@ -603,8 +595,7 @@
 {
 	LApic *la = &lapic;
 	
-	if(debug)
-		fprint(2, "CPU%d: TSC_DEADLINE=%llud\n", curcpuid, val);
+	dprint("CPU%d: TSC_DEADLINE=%llud\n", curcpuid, val);
 	
 	la->tsc_deadline = val;
 	
@@ -662,8 +653,7 @@
 	if(guest_tsc >= la->tsc_deadline){
 		vector = la->lvt_timer & 0xFF;
 		
-		if(debug)
-			fprint(2, "LAPIC%d: TSC deadline fired! vec=%d\n", curcpuid, vector);
+		dprint("LAPIC%d: TSC deadline fired! vec=%d\n", curcpuid, vector);
 		
 		la->tsc_deadline = 0;
 		la->tsc_deadline_armed = 0;
@@ -718,7 +708,7 @@
 vmtime_init(void)
 {
 	vmtime_initialized = 1;
-	fprint(2, "vmtime_init: using nanosec() directly\n");
+	dprint("vmtime_init: using nanosec() directly\n");
 }
 
 uvlong
--- a/virtio.c
+++ b/virtio.c
@@ -7,8 +7,10 @@
 #include <ip.h>		/* parseether() */
 #include <libsec.h>	/* genrandom() */
 
+static int local_devpipes[8][2] = {{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1}};
+
+
  
- 
 
 typedef struct VIODev VIODev;
 typedef struct VIOQueue VIOQueue;
@@ -73,9 +75,10 @@
 
 struct VIODev {
 	PCIDev *pci;
+	int isrstat ;  /* per-CPU instead of single  */
 	u32int devfeat, guestfeat;
 	u16int qsel;
-	u8int devstat, isrstat;
+	u8int devstat ;
 	VIOQueue *qu;
 	int nqu, allocqu;
 	u32int (*io)(int, u16int, u32int, int, VIODev *);
@@ -119,28 +122,8 @@
 void
 viosharedinit(void)
 {
-	int fd;
-	char buf[64];
-	
-	remove("#g/vmx.vio/ctl");
-	remove("#g/vmx.vio");
-	
-	fd = create("#g/vmx.vio", OREAD, DMDIR | 0777);
-	if(fd >= 0) close(fd);
-	
-	fd = open("#g/vmx.vio/ctl", OWRITE);
-	if(fd < 0)
-		sysfatal("viosharedinit: %r");
-	
-	snprint(buf, sizeof buf, "va 0x300004000 0x10000 sticky");
-	write(fd, buf, strlen(buf));
-	close(fd);
-	
-	vioshared = segattach(0, "vmx.vio", nil, 0x10000);
-	if(vioshared == (void*)-1)
-		sysfatal("viosharedinit segattach: %r");
-	
-	memset(vioshared, 0, sizeof(VIOShared));
+	vioshared = mkseg("vio", 0x300004000, 0x10000);
+    memset(vioshared, 0, sizeof(VIOShared));
 	ndevpipes = 0;
 
 	/* Allocate shared keyboard buffer */
@@ -180,7 +163,7 @@
 static void
 vioirq(VIODev *d, int val)
 {
-    int newstat;
+    int idx = d->devidx;
     
     lock(&d->isrlk);
     if(val != 0)
@@ -187,10 +170,13 @@
         d->isrstat |= val;
     else
         d->isrstat = 0;
-    newstat = d->isrstat;
-    unlock(&d->isrlk);  
+	 
+    unlock(&d->isrlk);
+
+
+    pciirq(d->pci, val != 0);
     
-    pciirq(d->pci, newstat != 0);
+//    pciirq(d->pci, val != 0);
 }
 
 static void *
@@ -319,8 +305,7 @@
         coherence();
     }
 
-    if(debug)
-        fprint(2, "VIOPUTBUF: usedidx=%d id=%d len=%d avail_flags=%#x\n",
+    dprint("VIOPUTBUF: usedidx=%d id=%d len=%d avail_flags=%#x\n",
                q->usedidx, b->idx, b->wptr, GET16(q->avail, 0));
 
 	int need_irq = (q->avail != nil && (GET16(q->avail, 0) & USEDNOIRQ) == 0);
@@ -420,15 +405,20 @@
 	return rc;
 }
 
+
+static int local_ackpipes[8][2];   /* worker completion */
+
 static void
 viowakeup(VIOQueue *q)
 {
-	char c = 1;
-	int idx;
-	
-	idx = q->d->devidx;
-	if(idx >= 0 && idx < ndevpipes && devpipes[idx][1] >= 0)
-		write(devpipes[idx][1], &c, 1);
+    char c = 1;
+    int idx = q->d->devidx;
+    
+    if(idx >= 0 && idx < 8 && vioshared->devpipes[idx][1] > 0){
+        write(vioshared->devpipes[idx][1], &c, 1);
+		 
+		//read(local_ackpipes[idx][0], &c, 1);
+	}
 }
 
 static void
@@ -535,17 +525,14 @@
     case 0x10012: return v->devstat;
     
     case 0x10013:  /* ISR read - atomic read and clear */
-        lock(&v->isrlk);
-        rc = v->isrstat;
-        v->isrstat = 0;
-        /* CRITICAL FIX: Call pciirq() while holding lock to prevent race
-         * with vioirq_() that could set isrstat and call pciirq() between
-         * our unlock and our pciirq(0) call. */
-        pciirq(v->pci, 0);
-        unlock(&v->isrlk);
-		if (debug)
-		fprint(2, "VIRTIO: ISR read, was %#x\n", rc);
-        return rc;
+	     lock(&v->isrlk);
+	     rc = v->isrstat;
+	     v->isrstat = 0; 
+	     unlock(&v->isrlk);
+		
+	     pciirq(v->pci, 0);
+	     dprint("VIRTIO: ISR read, was %#x\n", rc);
+	     return rc;
     }
     if(port >= 20 && v->io != nil)
         return v->io(isin, port - 20, val, sz, v);
@@ -560,9 +547,9 @@
 mkviodev_alloc(int nqu)
 {
     VIODev *d;
-    int i;
+    int i, idx;
     
-    if(ndevpipes >= MAXVIODEV)
+    if(vioshared->ndevpipes >= MAXVIODEV)
         sysfatal("mkviodev: too many virtio devices");
     
     d = vioalloc(sizeof(VIODev));
@@ -576,10 +563,11 @@
         d->qu[i].qidx = i;
     }
     
-    d->devidx = ndevpipes;
-    if(pipe(devpipes[ndevpipes]) < 0)
+	idx = vioshared->ndevpipes;
+    d->devidx = idx;
+    if(pipe(vioshared->devpipes[idx]) < 0)
         sysfatal("mkviodev pipe: %r");
-    ndevpipes++;
+    vioshared->ndevpipes++;
     
     return d;
 }
@@ -602,28 +590,19 @@
 static VIOBuf *
 viogetbuf_smp(VIOQueue *q, int wait)
 {
-	VIOBuf *b;
-	char c;
-	int idx;
-	
-	idx = q->d->devidx;
-	for(;;) {
-		b = viogetbuf(q, 0);
-		if(b != nil)
-			return b;
-		if(!wait)
-			return nil;
-
-		/* Debug: show we're about to block */
-        if(q->avail != nil)
-	if (debug)
-    fprint(2, "VIOBLK: blocking, availidx=%d guest_idx=%d livebuf=%d\n",
-           q->availidx, GET16(q->avail, 2), q->livebuf);
-
-		if(idx >= 0 && idx < ndevpipes && devpipes[idx][0] >= 0)
-			read(devpipes[idx][0], &c, 1);
- 
-	}
+    VIOBuf *b;
+    char c;
+    int idx = q->d->devidx;
+    
+    for(;;) {
+        b = viogetbuf(q, 0);
+        if(b != nil)
+            return b;
+        if(!wait)
+            return nil;
+        if(idx >= 0 && idx < 8 && vioshared->devpipes[idx][0] > 0)
+            read(vioshared->devpipes[idx][0], &c, 1);
+    }
 }
 
 VIOQueue *
@@ -945,8 +924,10 @@
 	
 	mkviodev_register(d, 0x1000, 0x020000, 1);
 	
-	proccreate(vionetrproc, d, 8192);
-	proccreate(vionetwproc, d, 8192);
+	vioshared->netdevs[vioshared->nnetdevs++] = d;
+
+//	proccreate(vionetrproc, d, 8192);
+//	proccreate(vionetwproc, d, 8192);
 	return 0;
 }
 
@@ -1002,9 +983,9 @@
     int rc, m;
     ulong n;
     vlong offset;
-    static Lock reqlock;
-    static uvlong reqcount = 0;
+    static Lock reqlock; 
     static uvlong writecount = 0;
+	static uvlong reqcount = 0;
     static uvlong readcount = 0;
     uvlong myreq, mywrite;
     
@@ -1012,13 +993,17 @@
     v = vp;
     q = &v->qu[0];
     
-    fprint(2, "VIOBLK: started fd=%d size=%llud sectors (%llud bytes)\n", 
+    dprint( "VIOBLK: started fd=%d size=%llud sectors (%llud bytes)\n", 
            v->blk.fd, v->blk.size, v->blk.size * 512ULL);
     if(v->blk.size == 0)
         fprint(2, "VIOBLK: WARNING: disk size is 0!\n");
     
+	int idx = v->devidx;
+	char c = 'w';
+
     for(;;){
-        b = viogetbuf_smp(q, 1); 
+  
+        b = viogetbuf_smp(q, 1);
         if(b == nil){
             vmerror("vioblkproc: viogetbuf: %r");
             threadexits("vioblkproc: viogetbuf: %r");
@@ -1034,30 +1019,25 @@
             VIOBuf *tb;
             int i = 0;
             ulong total_rd = 0, total_wr = 0;
-            if (debug)
-                fprint(2, "VIOBLK[%llud]: descriptor chain:\n", myreq);
+            dprint("VIOBLK[%llud]: descriptor chain:\n", myreq);
             for(tb = b; tb != nil; tb = tb->next, i++){
-                if (debug)
-                    fprint(2, "  desc[%d]: addr=%#llux len=%ud flags=%#ux", 
+                dprint("  desc[%d]: addr=%#llux len=%ud flags=%#ux", 
                            i, tb->addr, tb->len, tb->flags);
-                if(tb->flags & BUFCHAIN) fprint(2, " NEXT");
+                if(tb->flags & BUFCHAIN) dprint(" NEXT");
                 if(tb->flags & BUFWR) {
-                    fprint(2, " WRITE");
+                    dprint(" WRITE");
                     total_wr += tb->len;
                 } else {
                     total_rd += tb->len;
                 }
-                fprint(2, "\n");
+                dprint("\n");
             }
-            if (debug)
-                fprint(2, "  totals: readable=%lud writable=%lud\n", total_rd, total_wr);
+            dprint("  totals: readable=%lud writable=%lud\n", total_rd, total_wr);
         }
         
         ulong hdr_read = vioqread(b, cmd, sizeof(cmd));
         if(hdr_read < sizeof(cmd)){
-            if (debug)
-                fprint(2, "VIOBLK[%llud]: ERROR: header read failed: got %lud, need 16\n", 
-                       myreq, hdr_read);
+            dprint("VIOBLK[%llud]: ERROR: header read failed: got %lud, need 16\n", myreq, hdr_read);
             goto nope;
         }
         
@@ -1066,8 +1046,7 @@
         addr = GET64(cmd, 8);
         
         if(debug || myreq <= 5)
-            fprint(2, "VIOBLK[%llud]: type=%ud reserved=%#ux sector=%llud rptr_after_hdr=%ud\n", 
-                   myreq, type, reserved, addr, b->rptr);
+            dprint("VIOBLK[%llud]: type=%ud reserved=%#ux sector=%llud rptr_after_hdr=%ud\n", myreq, type, reserved, addr, b->rptr);
         
         switch(type){
         case 0:  /* READ */
@@ -1076,21 +1055,19 @@
             unlock(&reqlock);
             
             n = vioqrem(b, 1);
-            if(debug) fprint(2, "VIOBLK[%llud]: READ vioqrem(wr=1)=%lud\n", myreq, n);
+            dprint("VIOBLK[%llud]: READ vioqrem(wr=1)=%lud\n", myreq, n);
             
             if(n == 0){
-                if (debug)
-                    fprint(2, "VIOBLK[%llud]: ERROR: READ no writable space\n", myreq);
+                dprint("VIOBLK[%llud]: ERROR: READ no writable space\n", myreq);
                 ack = 1;
                 break;
             }
             n -= 1;
             
-            if(debug) fprint(2, "VIOBLK[%llud]: READ %lud bytes from sector %llud\n", myreq, n, addr);
+            dprint("VIOBLK[%llud]: READ %lud bytes from sector %llud\n", myreq, n, addr);
             
             if(addr * 512 + n > v->blk.size * 512){
-                if (debug)
-                    fprint(2, "VIOBLK[%llud]: ERROR: READ bounds failed\n", myreq);
+                dprint("VIOBLK[%llud]: ERROR: READ bounds failed\n", myreq);
                 ack = 1;
                 break;
             }
@@ -1101,14 +1078,12 @@
                 if(n < rc) rc = n;
                 rc = pread(v->blk.fd, buf, rc, offset);
                 if(rc < 0){
-                    if (debug)
-                        fprint(2, "VIOBLK[%llud]: ERROR: pread failed: %r\n", myreq);
+                    dprint("VIOBLK[%llud]: ERROR: pread failed: %r\n", myreq);
                     ack = 1;
                     break;
                 }
                 if(rc == 0){
-                    if (debug)
-                        fprint(2, "VIOBLK[%llud]: ERROR: pread EOF\n", myreq);
+                    dprint("VIOBLK[%llud]: ERROR: pread EOF\n", myreq);
                     ack = 1;
                     break;
                 }
@@ -1124,18 +1099,15 @@
             
             n = vioqrem(b, 0);
             if(debug || myreq <= 5)
-                fprint(2, "VIOBLK[%llud]: WRITE #%llud vioqrem(wr=0)=%lud sector=%llud\n", 
-                       myreq, mywrite, n, addr);
+                dprint("VIOBLK[%llud]: WRITE #%llud vioqrem(wr=0)=%lud sector=%llud\n", myreq, mywrite, n, addr);
             
             if(n == 0){
-                if (debug)
-                    fprint(2, "VIOBLK[%llud]: WARNING: WRITE with no data\n", myreq);
+                dprint("VIOBLK[%llud]: WARNING: WRITE with no data\n", myreq);
                 break;
             }
             
             if(addr * 512 + n > v->blk.size * 512){
-                if (debug)
-                    fprint(2, "VIOBLK[%llud]: ERROR: WRITE bounds failed\n", myreq);
+                dprint("VIOBLK[%llud]: ERROR: WRITE bounds failed\n", myreq);
                 ack = 1;
                 break;
             }
@@ -1149,14 +1121,12 @@
                 
                 rc = pwrite(v->blk.fd, buf, m, offset);
                 if(rc < 0){
-                    if (debug)
-                        fprint(2, "VIOBLK[%llud]: ERROR: pwrite failed: %r\n", myreq);
+                    dprint("VIOBLK[%llud]: ERROR: pwrite failed: %r\n", myreq);
                     ack = 1;
                     break;
                 }
                 if(rc < m){
-                    if (debug)
-                        fprint(2, "VIOBLK[%llud]: ERROR: short write %d < %d\n", myreq, rc, m);
+                    dprint("VIOBLK[%llud]: ERROR: short write %d < %d\n", myreq, rc, m);
                     ack = 1;
                     break;
                 }
@@ -1165,7 +1135,7 @@
             break;
             
         case 4:  /* FLUSH */
-            if(debug) fprint(2, "VIOBLK[%llud]: FLUSH\n", myreq);
+            dprint("VIOBLK[%llud]: FLUSH\n", myreq);
             ack = 0;
             break;
             
@@ -1187,19 +1157,53 @@
         vioqwrite(b, &ack, 1);
         
         if(debug || myreq <= 5)
-            fprint(2, "VIOBLK[%llud]: completing, ack=%d\n", myreq, ack);
+            dprint("VIOBLK[%llud]: completing, ack=%d\n", myreq, ack);
         
 		if(myreq % 1000 == 0)
-    		fprint(2, "REQ %llud\n", myreq);
+    		dprint("REQ %llud\n", myreq);
 
         vioputbuf(b);
         
         if(myreq % 1000 == 0)
-            if (debug)
-                fprint(2, "VIOBLK: %llud requests (%llud reads, %llud writes)\n",
-                       myreq, readcount, writecount);
+            dprint("VIOBLK: %llud requests (%llud reads, %llud writes)\n", myreq, readcount, writecount);
+
+		extern int wakepipe[MAXVCPU][2];
+		extern int hltpipe[MAXVCPU][2];
+
+ //		vioirq(v, 1);  /* set ISR here after all work done */
+ //       write(local_ackpipes[idx][1], &c, 1);  /* signal done */
+//		for(int i = 0; i < nvcpu; i++) 
+//	        	write(wakepipe[i][1], &c, 1);
+
     }
 }
+
+ 
+
+void
+virtio_start_workers(void)
+{
+    int i;
+
+    /* Create LOCAL pipes for this process */
+    for(i = 0; i < vioshared->nblkdevs; i++){
+		 
+        if(pipe(local_devpipes[i]) < 0)
+            sysfatal("virtio pipe: %r");
+		if(pipe(local_ackpipes[i]) < 0)
+			sysfatal("virtio pipe: %r");
+        proccreate(vioblkproc, vioshared->blkdevs[i], 131072);
+    }
+    
+    for(i = 0; i < vioshared->nnetdevs; i++){
+        if(pipe(local_devpipes[vioshared->nblkdevs + i]) < 0)
+            sysfatal("virtio pipe: %r");
+        proccreate(vionetrproc, vioshared->netdevs[i], 8192);
+        proccreate(vionetwproc, vioshared->netdevs[i], 8192);
+    }
+}
+
+
 int
 mkvioblk(char *fn)
 {
@@ -1230,7 +1234,7 @@
     
     size = filesize >> 9;
     
-    fprint(2, "VIOBLK: opened '%s' fd=%d filesize=%lld size=%llud sectors\n",
+    dprint("VIOBLK: opened '%s' fd=%d filesize=%lld size=%llud sectors\n",
            fn, fd, filesize, size);
     
     if(size == 0){
@@ -1254,8 +1258,10 @@
     
     mkviodev_register(d, 0x1001, 0x018000, 2);
 
-    for(int i = 0; i < 1; i++)
-	    proccreate(vioblkproc, d, 131072);
+	vioshared->blkdevs[vioshared->nblkdevs++] = d;
+
+//    for(int i = 0; i < 1; i++)
+//	    proccreate(vioblkproc, d, 131072);
 
     return 0;
 }
--- a/vmx.c
+++ b/vmx.c
@@ -6,11 +6,76 @@
 #include "dat.h"
 #include "fns.h"
 
+char *vmx_segprefix;
+
 static char *uartcfg[2];  
 /* Saved UART config from -c/-C options */
 
    extern MouseShared *mouseshared;
 
+void *
+mkseg(char *name, uintptr va, ulong size)
+{
+    char path[64];
+    char buf[64];
+    int fd;
+    void *p;
+    
+    /* Remove old segment if exists */
+    snprint(path, sizeof path, "#g/%s.%s/ctl", vmx_segprefix, name);
+    remove(path);
+    snprint(path, sizeof path, "#g/%s.%s", vmx_segprefix, name);
+    remove(path);
+    
+    /* Create directory */
+    fd = create(path, OREAD|ORCLOSE, DMDIR | 0777);
+    //if(fd >= 0) close(fd);
+    
+    /* Configure segment */
+    snprint(path, sizeof path, "#g/%s.%s/ctl", vmx_segprefix, name);
+    fd = open(path, OWRITE);
+    if(fd < 0)
+        sysfatal("mkseg %s: open ctl: %r", name);
+    
+    snprint(buf, sizeof buf, "va %#p %#lux sticky", va, size);
+    if(write(fd, buf, strlen(buf)) < 0)
+        sysfatal("mkseg %s: write ctl: %r", name);
+    close(fd);
+    
+    /* Attach */
+    snprint(path, sizeof path, "%s.%s", vmx_segprefix, name);
+    p = segattach(0, path, nil, size);
+    if(p == (void*)-1)
+        sysfatal("mkseg %s: segattach: %r", name);
+    
+    return p;
+}
+
+void
+rmseg(char *name)
+{
+    char path[64];
+    
+    snprint(path, sizeof path, "#g/%s.%s/ctl", vmx_segprefix, name);
+    remove(path);
+    snprint(path, sizeof path, "#g/%s.%s", vmx_segprefix, name);
+    remove(path);
+}
+
+void
+dprint(char *fmt, ...)
+{
+    va_list arg;
+    
+    if(!debug)
+        return;
+    
+    fprint(2, "CPU%d: ", curcpuid);
+    va_start(arg, fmt);
+    vfprint(2, fmt, arg);
+    va_end(arg);
+}
+
    void
     mousesharedinit(void)
     {
@@ -113,13 +178,15 @@
 u32int preempt_shift;  
 int timerid;
 
+char name[128];
+
 void
 vmxsetup(void)
 {
 	static char buf[128];
-	static char name[128];
-	int rc;
 	
+	int rc;
+	close(ctlfd); // When forking
 	ctlfd = open("#X/clone", ORDWR|ORCLOSE);
 	if(ctlfd < 0) sysfatal("open: %r");
 	rc = read(ctlfd, name, sizeof(name) - 1);
@@ -126,8 +193,8 @@
 	if(rc < 0) sysfatal("read: %r");
 	name[rc] = 0;
 	if(segname == nil){
-		segname = smprint("vm.%s", name);
-		segrclose = ORCLOSE;
+   		segname = smprint("vm.%d.%s", getpid(), name);
+    	segrclose = ORCLOSE;
 	}
 	
 	snprint(buf, sizeof(buf), "#X/%s/regs", name);
@@ -147,8 +214,11 @@
 	preempt_shift = (u32int)rget("preemptshift");
 	
 	/* DEBUG - always print these */
-	fprint(2, "VMX: preempt_shift=%d tsc_freq=%llud\n", preempt_shift, _tos->cyclefreq);
+	dprint("VMX: preempt_shift=%d tsc_freq=%llud\n", preempt_shift, _tos->cyclefreq);
 
+	snprint(buf, sizeof(buf), "#X/%s/ctl", name);
+	open(buf, OREAD|ORCLOSE); // remove ctl when process exits
+
 }
 
 enum { RCENT = 256 };
@@ -636,11 +706,15 @@
     timers_advance(); 
     launch(); 
     
+	/* Drain garbage from before VM started */
+	kbdshared->r = kbdshared->w;
+
     /* Set channel pointers once */
     a[AWAIT].c = waitch;
     a[ANOTIF].c = notifch;
     
 	static int count = 0;
+ 
 
     for(;;){
  
@@ -837,6 +911,10 @@
 	int newwin = 0;
 	int i;
 
+	 
+
+	vmx_segprefix = smprint("vmx.%d", getpid());
+
 	quotefmtinstall();
 	mainid = threadid();
 	cpuidinit();
@@ -919,6 +997,8 @@
 		mkregion(fbaddr, fbaddr+fbsz, REGALLOC|REGRWX);
 	}
 
+	
+
 	vmxsetup();
  	vmtime_init(); 
  
@@ -1000,6 +1080,11 @@
 	if(srvname != nil) init9p(srvname);
 	if(kconfig != nil) kconfig();
 
+    virtio_start_workers();  /* Start workers for CPU0 */
+
+	vmx_cleanup_init();
+
 	runloop();
+	 
 	exits(nil);
 }
-- 
⑨