shithub: vmxsmp

ref: e0e889fb69a6d4d3f2332244ef4f79f900a66d84
dir: /vmx.c/

View raw version
#include <u.h>
#include <libc.h>
#include <thread.h>
#include <tos.h>
#include <draw.h>
#include "dat.h"
#include "fns.h"

char *vmx_segprefix;

static char *uartcfg[2];  
/* Saved UART config from -c/-C options */

   extern MouseShared *mouseshared;

void *
mkseg(char *name, uintptr va, ulong size)
{
    char path[64];
    char buf[64];
    int fd;
    void *p;
    
    /* Remove old segment if exists */
    snprint(path, sizeof path, "#g/%s.%s/ctl", vmx_segprefix, name);
    remove(path);
    snprint(path, sizeof path, "#g/%s.%s", vmx_segprefix, name);
    remove(path);
    
    /* Create directory */
    fd = create(path, OREAD|ORCLOSE, DMDIR | 0777);
    //if(fd >= 0) close(fd);
    
    /* Configure segment */
    snprint(path, sizeof path, "#g/%s.%s/ctl", vmx_segprefix, name);
    fd = open(path, OWRITE);
    if(fd < 0)
        sysfatal("mkseg %s: open ctl: %r", name);
    
    snprint(buf, sizeof buf, "va %#p %#lux sticky", va, size);
    if(write(fd, buf, strlen(buf)) < 0)
        sysfatal("mkseg %s: write ctl: %r", name);
    close(fd);
    
    /* Attach */
    snprint(path, sizeof path, "%s.%s", vmx_segprefix, name);
    p = segattach(0, path, nil, size);
    if(p == (void*)-1)
        sysfatal("mkseg %s: segattach: %r", name);
    
    return p;
}

void
rmseg(char *name)
{
    char path[64];
    
    snprint(path, sizeof path, "#g/%s.%s/ctl", vmx_segprefix, name);
    remove(path);
    snprint(path, sizeof path, "#g/%s.%s", vmx_segprefix, name);
    remove(path);
}

void
dprint(char *fmt, ...)
{
    va_list arg;
    
    if(!debug)
        return;
    
    fprint(2, "CPU%d: ", curcpuid);
    va_start(arg, fmt);
    vfprint(2, fmt, arg);
    va_end(arg);
}

   void
    mousesharedinit(void)
    {
        extern VIOShared *vioshared;
        
        if(vioshared == nil)
            sysfatal("mousesharedinit: vioshared not initialized");
        
        mouseshared = (MouseShared*)&vioshared->data[vioshared->alloc];
        vioshared->alloc += sizeof(MouseShared);
        
        if(vioshared->alloc > sizeof(vioshared->data))
            sysfatal("mousesharedinit: out of shared memory");
        
        memset(mouseshared, 0, sizeof(MouseShared));
    }
 

char *segname;
int segrclose;
Region *mmap;
int ctlfd, regsfd, mapfd, waitfd;
Channel *waitch, *notifch;
enum { MSEC = 1000*1000 } ;
int getexit, state, debug;

extern int kbdpipe[2];
extern int uartpipe[2][2]; 
extern UartShared *uartshared[2];
extern void kbdreader(void*);
extern void uartreader(void*);

void *
emalloc(ulong sz)
{
	void *v;
	
	v = malloc(sz);
	if(v == nil)
		sysfatal("malloc: %r");
	memset(v, 0, sz);
	setmalloctag(v, getcallerpc(&sz));
	return v;
}

void
vmerror(char *fmt, ...)
{
	Fmt f;
	char buf[256];
	va_list arg;
	
	fmtfdinit(&f, 2, buf, sizeof buf);
	va_start(arg, fmt);
	fmtvprint(&f, fmt, arg);
	va_end(arg);
	fmtprint(&f, "\n");
	fmtfdflush(&f);
}

int
ctl(char *fmt, ...)
{
	va_list va;
	int rc;
	char err [ERRMAX];

repeat:

	va_start(va, fmt);
	rc = vfprint(ctlfd, fmt, va);
	va_end(va);

	if (rc < 0) {
		 
		rerrstr(err, sizeof(err)); 
        if(strstr(err, "ipi") == nil){
			goto repeat;
		} 
	}

	return rc;
}

void
modregion(Region *r)
{
	if(r->segname == nil){
		if(fprint(mapfd, "--- wb %#ullx %#ullx\n", (uvlong)r->start, (uvlong)r->end) < 0)
			vmerror("updating memory map: %r");
	}else
		if(fprint(mapfd, "%c%c%c wb %#ullx %#ullx %s %#ullx\n",
			(r->type & REGR) != 0 ? 'r' : '-',
			(r->type & REGW) != 0 ? 'w' : '-',
			(r->type & REGX) != 0 ? 'x' : '-',
			(uvlong)r->start, (uvlong)r->end, r->segname, (uvlong)r->segoff) < 0)
			vmerror("updating memory map: %r");
}

u32int preempt_shift;  
int timerid;

char name[128];

void
vmxsetup(void)
{
	static char buf[128];
	
	int rc;
	close(ctlfd); // When forking
	ctlfd = open("#X/clone", ORDWR|ORCLOSE);
	if(ctlfd < 0) sysfatal("open: %r");
	rc = read(ctlfd, name, sizeof(name) - 1);
	if(rc < 0) sysfatal("read: %r");
	name[rc] = 0;
	if(segname == nil){
   		segname = smprint("vm.%d.%s", getpid(), name);
    	segrclose = ORCLOSE;
	}
	
	snprint(buf, sizeof(buf), "#X/%s/regs", name);
	regsfd = open(buf, ORDWR);
	if(regsfd < 0) sysfatal("open: %r");
	
	snprint(buf, sizeof(buf), "#X/%s/map", name);
	mapfd = open(buf, OWRITE|OTRUNC);
	if(mapfd < 0) sysfatal("open: %r");
	
	snprint(buf, sizeof(buf), "#X/%s/wait", name);
	waitfd = open(buf, OREAD);
	if(waitfd < 0) sysfatal("open: %r");
	
	/* Read preemption timer shift from kernel */
	rcload();
	preempt_shift = (u32int)rget("preemptshift");
	
	/* DEBUG - always print these */
	dprint("VMX: preempt_shift=%d tsc_freq=%llud\n", preempt_shift, _tos->cyclefreq);

	snprint(buf, sizeof(buf), "#X/%s/ctl", name);
	open(buf, OREAD|ORCLOSE); // remove ctl when process exits

}

enum { RCENT = 256 };
static char *rcname[RCENT];
static uvlong rcval[RCENT];
uvlong rcvalid[(RCENT+63)/64], rcdirty[(RCENT+63)/64];

static int
rclookup(char *n)
{
	int i;
	
	for(i = 0; i < RCENT; i++)
		if(rcname[i] != nil && strcmp(n, rcname[i]) == 0)
			return i;
	return -1;
}

char *
rcflush(int togo)
{
	int i, j;
	static char buf[4096];
	char *p, *e;
	uvlong v;
	char err [4096];
	
	repeat:

	p = buf;
	e = buf + sizeof(buf);
	*p = 0;
	for(i = 0; i < (RCENT+63)/64; i++){
		if(v = rcdirty[i], v != 0){
			for(j = 0; j < 64; j++)
				if((v>>j & 1) != 0)
					p = seprint(p, e, "%s%c%#ullx%c", rcname[i*64+j], togo?'=':' ', rcval[i*64+j], togo?';':'\n');
			rcdirty[i] = 0;
		}
		rcvalid[i] = 0;
	}
	if(!togo && p != buf && write(regsfd, buf, p - buf) < p - buf){
		rerrstr(err, sizeof(err));
        if(strstr(err, "ipi") == nil) 
			goto repeat;
		else
			sysfatal("rcflush: write: %r");
	}
	return p != buf ? buf : nil;
}

void
rcload(void)
{
	static char buf[4096];
	char *p, *q, *f[2];
	int nf;
	int i, rc;
	char err[4096];

	rcflush(0);


repeat:
	rc = pread(regsfd, buf, sizeof(buf) - 1, 0);
	if(rc < 0) {
		rerrstr(err, sizeof(err));
		if(strstr(err, "ipi") == nil)
			goto repeat;
		else
			sysfatal("rcload: pread: %r");
	}
	buf[rc] = 0;
	p = buf;
	for(i = 0; i < nelem(rcname); i++){
		q = strchr(p, '\n');
		if(q == nil) break;
		*q = 0;
		nf = tokenize(p, f, nelem(f));
		p = q + 1;
		if(nf < 2) break;
		rcname[i] = f[0];
		rcval[i] = strtoull(f[1], nil, 0);
		rcvalid[i>>6] |= 1ULL<<(i&63);
	}
	for(; i < nelem(rcname); i++){
		rcname[i] = nil;
		rcvalid[i>>6] &= ~(1ULL<<(i&63));
	}
}

uvlong
rget(char *reg)
{
	int i;

	i = rclookup(reg);
	if(i < 0 || (rcvalid[i>>6]>>i&1) == 0){
		rcload();
		i = rclookup(reg);
		if(i < 0) sysfatal("unknown register %s", reg);
	}
	return rcval[i];
}

void
rpoke(char *reg, uvlong val, int clean)
{
	int i;

	i = rclookup(reg);
	if(i >= 0){
		if((rcvalid[i>>6]>>(i&63)&1) != 0 && rcval[i] == val) return;
		goto goti;
	}
	for(i = 0; i < nelem(rcname); i++)
		if(rcname[i] == nil){
			rcname[i] = reg;
			break;
		}
	assert(i < nelem(rcname));
goti:
	rcval[i] = val;
	rcvalid[i>>6] |= 1ULL<<(i&63);
	if(!clean)
		rcdirty[i>>6] |= 1ULL<<(i&63);
}

uvlong
rgetsz(char *reg, int sz)
{
	switch(sz){
	case 1: return (u8int)rget(reg);
	case 2: return (u16int)rget(reg);
	case 4: return (u32int)rget(reg);
	case 8: return rget(reg);
	default:
		vmerror("invalid size operand for rgetsz");
		assert(0);
		return 0;
	}
}

void
rsetsz(char *reg, uvlong val, int sz)
{
	switch(sz){
	case 1: rset(reg, (u8int)val | rget(reg) & ~0xffULL); break;
	case 2: rset(reg, (u16int)val | rget(reg) & ~0xffffULL); break;
	case 4: rset(reg, (u32int)val); break;
	case 8: rset(reg, val); break;
	default:
		vmerror("invalid size operand for rsetsz");
		assert(0);
	}
}

Region *
mkregion(u64int pa, u64int end, int type)
{
	Region *r, *s, **rp;

	r = emalloc(sizeof(Region));
	if(end < pa) sysfatal("end of region %p before start of region %#p", (void*)end, (void*)pa);
	if((pa & BY2PG-1) != 0 || (end & BY2PG-1) != 0) sysfatal("address %#p not page aligned", (void*)pa);
	r->start = pa;
	r->end = end;
	r->type = type;
	for(s = mmap; s != nil; s = s->next)
		if(!(pa < s->start && end < s->end || pa >= s->start && pa >= s->end))
			sysfatal("region %#p-%#p overlaps region %#p-%#p", (void*)pa, (void*)end, (void*)s->start, (void*)s->end);
	for(rp = &mmap; (*rp) != nil && (*rp)->start < end; rp = &(*rp)->next)
		;
	r->next = *rp;
	*rp = r;
	return r;
}

Region *
regptr(u64int addr)
{
	Region *r;

	for(r = mmap; r != nil; r = r->next)
		if(addr >= r->start && addr < r->end)
			return r;
	return nil;
}

void *
gptr(u64int addr, u64int len)
{
	Region *r;

	if(addr + len < addr)
		return nil;
	for(r = mmap; r != nil; r = r->next)
		if(addr >= r->start && addr < r->end){
			if(addr + len > r->end)
				return nil;
			return (uchar *) r->v + (addr - r->start);
		}
	return nil;
}

uintptr
gpa(void *v)
{
	Region *r;

	for(r = mmap; r != nil; r = r->next)
		if(v >= r->v && v < r->ve)
			return (uchar *) v - (uchar *) r->v + r->start;
	return -1;
}

uintptr
gavail(void *v)
{
	Region *r;
	
	for(r = mmap; r != nil; r = r->next)
		if(v >= r->v && v < r->ve)
			return (uchar *) r->ve - (uchar *) v;
	return 0;
}

void *
gend(void *v)
{
	return (u8int *) v + gavail(v);
}

static void
mksegment(char *sn)
{
	uintptr sz;
	int fd;
	Region *r;
	char buf[256];
	u8int *gmem, *p;

	sz = BY2PG; /* temporary page */
	sz += 256*1024; /* vga */
	for(r = mmap; r != nil; r = r->next){
		if((r->type & REGALLOC) == 0)
			continue;
		r->segname = sn;
		if(sz + (r->end - r->start) < sz)
			sysfatal("out of address space");
		sz += r->end - r->start;
	}
	gmem = segattach(0, sn, nil, sz);
	if(gmem == (void*)-1){
		snprint(buf, sizeof(buf), "#g/%s", sn);
		fd = create(buf, OREAD|segrclose, DMDIR | 0777);
		if(fd < 0) sysfatal("create: %r");
		snprint(buf, sizeof(buf), "#g/%s/ctl", sn);
		fd = open(buf, OWRITE|OTRUNC);
		if(fd < 0) sysfatal("open: %r");
		snprint(buf, sizeof(buf), "va %#ullx %#ullx sticky", 0x10000000ULL, (uvlong)sz);
		if(write(fd, buf, strlen(buf)) < 0) sysfatal("write: %r");
		close(fd);
		gmem = segattach(0, sn, nil, sz);
		if(gmem == (void*)-1) sysfatal("segattach: %r");
	}else{
		memset(gmem, 0, sz > 1<<24 ? 1<<24 : sz);
	}
	p = gmem;
	for(r = mmap; r != nil; r = r->next){
		if(r->segname == nil) continue;
		r->segoff = p - gmem;
		r->v = p;
		p += r->end - r->start;
		r->ve = p;
	}
	/* vga */
	r = regptr(0xa0000);
	r->segoff = p - gmem;
	r->v = p;
	p += 256*1024;
	r->ve = p;

	for(r = mmap; r != nil; r = r->next)
		modregion(r);

}

void
postexc(char *name, vlong code)
{
	if(code >= 0){
		if(ctl("exc %s,%#ux", name, (u32int)code) < 0)
			sysfatal("ctl(postexc): %r");
	}else
		if(ctl("exc %s", name) < 0)
			sysfatal("ctl(postexc): %r");
}
 
vlong cached_tscoff;

extern void ipi_poll(void);

/* The system is guaranteed a VM exit between these bounds */

#define MIN_TRIGGER 1000000LL    /* 0.1ms */
#define MAX_TRIGGER 10000000LL  /* 10ms */

int wakepipe[MAXVCPU][2];

extern u32int preempt_shift;

void
preempt_cap(vlong min_ns, vlong max_ns)
{
    vlong nearest, now, delta;
    uvlong tsc_freq, ticks;
    extern vlong timer_nearest(void);

    nearest = timer_nearest();
    now = nanosec();
    delta = nearest - now;
    
    /* If deadline is in the past or too far, cap it */
    if(delta <= min_ns)
        delta = min_ns;  
    else if(delta > max_ns)
        delta = max_ns;
    
    tsc_freq = _tos->cyclefreq;
    if(tsc_freq == 0)
        tsc_freq = 2900000000ULL;
    
    ticks = (delta * tsc_freq / 1000000000ULL) >> preempt_shift;
    
    if(ticks > 0xFFFFFFFF)
        ticks = 0xFFFFFFFF;
    if(ticks == 0)
        ticks = 1;
    
    ctl("preempt %ud", (u32int)ticks);
}

 
void
launch(void)
{
      
    /* Clear halt_tsc - we don't adjust tscoff anymore since
     * all VMM timers use raw host TSC consistently */
  	
	rcflush(0); 
	if(ctl("go") < 0)
		sysfatal("go");
 
    getexit++;
}

static void
waitproc(void *)
{
	static char buf[512];
	char *p;
	int rc;

	threadsetname("waitexit");
	for(;;){
		rc = read(waitfd, buf, sizeof(buf) - 1);
		if(rc < 0)
			sysfatal("read: %r");
		buf[rc] = 0;
		p = strchr(buf, '\n');
		if(p != nil) *p = 0;
		sendp(waitch, strdup(buf));
	}
}
   
inline int
has_pending_work(void)
{
    extern IpiQueue *ipiqueue;
    extern u32int irr_bitmap[8];
    
    /* Quick non-locking check - may have false negatives, that's OK */
    if(irr_bitmap[0] | irr_bitmap[1] | irr_bitmap[2] | irr_bitmap[3] |
       irr_bitmap[4] | irr_bitmap[5] | irr_bitmap[6] | irr_bitmap[7])
        return 1;
    
    if(ipiqueue != nil && ipiqueue->cpu[curcpuid].wake)
        return 1;
    
    return 0;
}

Lock statelock;

#define VLONG_MAX 0x7FFFFFFFFFFFFFFFLL

int hltpipe[MAXVCPU][2];

void awakeproc(void *)
{
 

    for(;;) {
        extern vlong timer_nearest(void);
        vlong now, nearest, sleep_ms;
		
		sleep(10);	 
	    write(wakepipe[curcpuid][1], "w", 1);

    }
}
 
static void
wakeproc(void *)
{
    char c;
    int n;

    threadsetname("wakepipe");
    for(;;){
        n = read(wakepipe[curcpuid][0], &c, 1);
        if(n == 1){
            lock(&statelock);
            if(state == VMHALT)
                write(hltpipe[curcpuid][1], "w", 1);
			else 
				preempt_cap(MIN_TRIGGER, MIN_TRIGGER); 
			
            unlock(&statelock);
        } else if(n <= 0){
            sleep(10);
        }
    }
}

/* Batch channel draining */
static inline int
drain_channels(void)
{
    char *waitmsg;
    VmxNotif notif;
    int processed = 0;
    
    /* Drain up to 8 wait messages at once */
    while(processed < 8 && nbrecv(waitch, &waitmsg)){
        getexit--;
        processexit(waitmsg);
        free(waitmsg);
        processed++;
    }
    
    /* Drain up to 8 notifications at once */
    while(processed < 16 && nbrecv(notifch, &notif)){
        if(notif.f != nil)
            notif.f(notif.arg);
        processed++;
    }
    
    return processed;
}

extern void inject_pending_irq(void);
extern void uartkick(UART *);

void
runloop(void)
{
    char *waitmsg;
    VmxNotif notif;
    
    enum { AWAIT, ANOTIF, NALT };
    Alt a[] = {
        [AWAIT]  = { nil, &waitmsg, CHANRCV },
        [ANOTIF] = { nil, &notif,   CHANRCV },
        [NALT]   = { nil, nil,      CHANEND },
    };
    
    proccreate(waitproc, nil, 4096);
	proccreate(wakeproc, nil, 8192);
	proccreate(awakeproc, nil, 8192);
  
    pvclock_update(curcpuid);
    timers_advance(); 
    launch(); 
    
	/* Drain garbage from before VM started */
	kbdshared->r = kbdshared->w;

    /* Set channel pointers once */
    a[AWAIT].c = waitch;
    a[ANOTIF].c = notifch;
    
	static int count = 0;
 

    for(;;){
 
        /* BLOCK here until something arrives - no CPU burn */
        switch(alt(a)){
        case AWAIT:
            getexit--;
            processexit(waitmsg);
            free(waitmsg);
            break;
            
        case ANOTIF:
            if(notif.f != nil)
                notif.f(notif.arg);
            break;
        }
      
		

        /* Only do work when VM has exited */
        if(getexit == 0){
           
			/* Timers -> lowest priority */
   
			timers_advance();
 

			/* Non-timer interrupts -> medium */

			ipi_poll();
			 

			/* Keyboard/UART on CPU0 highest */

			if (curcpuid == 0){
	        	i8042kick(nil);
        		if(uart != nil){
               		uartkick(&uart[0]);
                	uartkick(&uart[1]);
        		}
	 		}

			if(irr_pending())
				inject_pending_irq();
 
            
		  	preempt_cap(MIN_TRIGGER, MAX_TRIGGER); 
            launch();

			
        }  
    }
}
static int mainid;

void
sendnotif(void (*f)(void *), void *arg)
{
    VmxNotif notif = {f, arg};
    
    if(threadid() == mainid)
        f(arg);
    else{
        /* Non-blocking send with retry to avoid blocking workers */
        while(nbsend(notifch, &notif) == 0){
			sleep(1);
            yield();  /* Give main thread a chance to receive */
        }
    }
}

extern void vgainit(int);
extern void pciinit(void);
extern void pcibusmap(void);
extern void cpuidinit(void);
extern void vgafbparse(char *);
extern void init9p(char *);

int cmdlinen;
char **cmdlinev;
int bootmodn;
char **bootmod;

static uvlong
siparse(char *s)
{
	uvlong l;
	char *p;
	
	l = strtoull(s, &p, 0);
	switch(*p){
	case 'k': case 'K': p++; l *= 1<<10; break;
	case 'm': case 'M': p++; l *= 1<<20; break;
	case 'g': case 'G': p++; l *= 1<<30; break;
	}
	if(*p != 0) sysfatal("invalid argument: %s", s);
	return l;
}

static void
setiodebug(char *s)
{
	char *p;
	int n, m, neg;
	extern u32int iodebug[32];
	
	do{
		if(neg = *s == '!')
			s++;
		n = strtoul(s, &p, 0);
		if(s == p)
no:			sysfatal("invalid iodebug argument (error at %#q)", s);
		if(n >= sizeof(iodebug)*8)
range:			sysfatal("out of iodebug range (0-%#ux)", sizeof(iodebug)*8-1);
		s = p + 1;
		if(*p == '-'){
			m = strtoul(s, &p, 0);
			if(m >= sizeof(iodebug)*8)
				goto range;
			if(s == p || m < n) goto no;
			s = p + 1;
		}else
			m = n;
		for(; n <= m; n++)
			if(neg)
				iodebug[n>>5] &= ~(1<<(n&31));
			else
				iodebug[n>>5] |= 1<<(n&31);
	}while(*p == ',');
	if(*p != 0) goto no;
}

static void
usage(void)
{
	char *blanks, *p;
	
	blanks = strdup(argv0);
	for(p = blanks; *p != 0; p++)
		*p = ' ';
	fprint(2, "usage: %s [ -M mem ] [ -c com1rd[,com1wr] ] [ -C com2rd[,com2r] ] [ -n nic ]\n", argv0);
	fprint(2, "       %s [ -d blockfile ] [ -m module ] [ -v|-w vga ] [ -9 srv ] kernel [ args ... ]\n", blanks);
	threadexitsall("usage");
}

void (*kconfig)(void);

uint nvcpu = 4;
uint curcpuid = 0; 
LApic lapic; 
IOApic *ioapic;
 

void
wakepipe_init(void)
{
    int i;
    
    for(i = 0; i < nvcpu; i++){
        if(pipe(wakepipe[i]) < 0)
            sysfatal("wakepipe[%d]: %r", i);
        
        /* Store write-end in shared memory so everyone knows it */
        ipiqueue->wakefds[i] = wakepipe[i][1];
    }
}
 

void hltpipe_init(void)
{
    for(int i = 0; i < nvcpu; i++){
        if(pipe(hltpipe[i]) < 0)
            sysfatal("hltpipe[%d]: %r", i);

		
        /* Store write-end in shared memory so everyone knows it */
        ipiqueue->hltfs[i] = hltpipe[i][1];

    }
}

int (*edev[16])(char *);
char *edevt[16];
char *edevaux[16];
int edevn;
int uarttxpipe[2][2]; 
void
threadmain(int argc, char **argv)
{
	extern void pcisharedinit(void);
	static uvlong gmemsz = 64*1024*1024;
	static char *srvname;
	extern uintptr fbsz, fbaddr;
	int newwin = 0;
	int i;

	 

	vmx_segprefix = smprint("vmx.%d", getpid());

	quotefmtinstall();
	mainid = threadid();
	cpuidinit();
	waitch = chancreate(sizeof(char *), 32);
	notifch = chancreate(sizeof(VmxNotif), 256);

	ARGBEGIN {
	case 'm':
		bootmod = realloc(bootmod, (bootmodn + 1) * sizeof(char *));
		bootmod[bootmodn++] = strdup(EARGF(usage()));
		break;
	case 's':
		segname = strdup(EARGF(usage()));
		segrclose = 0;
		break;
    case 'c':
        uartcfg[0] = strdup(EARGF(usage()));  /* Save config, don't call uartinit yet */
        break;
    case 'C':
        uartcfg[1] = strdup(EARGF(usage()));  /* Save config, don't call uartinit yet */
        break;
	case 'n':
		assert(edevn < nelem(edev));
		edev[edevn] = mkvionet;
		edevt[edevn] = "virtio network";
		edevaux[edevn++] = strdup(EARGF(usage()));
		break;
	case 'd':
		assert(edevn < nelem(edev));
		edevaux[edevn] = strdup(EARGF(usage()));
		edev[edevn] = mkvioblk;
		edevt[edevn] = "virtio block";
		edevn++;
		break;
	case 'D':
		debug++;
		break;
	case 'M':
		gmemsz = siparse(EARGF(usage()));
		if(gmemsz != (uintptr) gmemsz) sysfatal("too much memory for address space");
		break;
	case 'x':
		nvcpu = siparse(EARGF(usage()));
		break;
	case 'w':
		newwin = 1;
	case 'v':
		vgafbparse(EARGF(usage()));
		break;
	case '9':
		if(srvname != nil) usage();
		srvname = EARGF(usage());
		break;
	case L'ι':
		setiodebug(EARGF(usage()));
		break;
	default:
		usage();
	} ARGEND;
	if(argc < 1) usage();
	cmdlinen = argc - 1;
	cmdlinev = argv + 1;
	
	if(gmemsz < 1<<20) sysfatal("640 KB of RAM is not enough for everyone");
	mkregion(0, 0xa0000, REGALLOC|REGFREE|REGRWX);
	mkregion(0xa0000, 0xc0000, REGALLOC|REGRWX);
	mkregion(0xc0000, 0x100000, REGALLOC|REGRES|REGRWX);
	mkregion(0xFEC00000, 0xFEC01000, REGALLOC); /* I/O APIC */
	mkregion(0xFED00000, 0xFED01000, REGALLOC); /* HPET */
	mkregion(0xFEE00000, 0xFEE01000, REGALLOC); /* LAPIC */

	if(fbsz != 0 && fbaddr < gmemsz){
		mkregion(0x100000, fbaddr, REGALLOC|REGFREE|REGRWX);
		mkregion(fbaddr + fbsz, gmemsz, REGALLOC|REGFREE|REGRWX);
	}else
		mkregion(0x100000, gmemsz, REGALLOC|REGFREE|REGRWX);
	if(fbsz != 0){
		if(fbaddr < 1<<20) sysfatal("framebuffer must not be within first 1 MB");
		if(fbaddr != (u32int) fbaddr || (u32int)(fbaddr+fbsz) < fbaddr) sysfatal("framebuffer must be within first 4 GB");
		mkregion(fbaddr, fbaddr+fbsz, REGALLOC|REGRWX);
	}

	

	vmxsetup();
 	vmtime_init(); 
 
	mksegment(segname);

 

   

	extern void lapic_timer_init(void);

    ioapic_init();
    ipiqueueinit(); 
	hltpipe_init();
	wakepipe_init();
	lapic_timer_init();
	extern void viosharedinit(void);
	viosharedinit();
	pit_init();
	rtc_init();
    hpet_init(); 

    pvclock_init();   
	pcisharedinit();

	extern void vionotifyinit(void);
	extern void viosharedinit(void);
	extern void vgasharedinit(void);
	extern void mousesharedinit(void);
	extern void mouseps2init(void);
	vionotifyinit();
 
	if(pipe(kbdpipe) < 0)
		sysfatal("kbdpipe: %r");
	if(pipe(uartpipe[0]) < 0)
		sysfatal("uartpipe0: %r");
	if(pipe(uartpipe[1]) < 0)
		sysfatal("uartpipe1: %r");
	if(pipe(uarttxpipe[0]) < 0)
    	sysfatal("uarttxpipe0: %r");
	if(pipe(uarttxpipe[1]) < 0)
    	sysfatal("uarttxpipe1: %r");
   if(pipe(mousepipe) < 0)
       sysfatal("mousepipe: %r");
   
 
	vgasharedinit();
	mousesharedinit();
	mouseps2init();  /* Initialize PS/2 mouse state in shared memory */
	pciinit();

    vgainit(newwin);
    extern void mousereader(void *);
    proccreate(mousereader, nil, 8192);

    /* Initialize UARTs NOW - after uart is allocated */
    if(uartcfg[0] != nil)
        uartinit(0, uartcfg[0]);
    if(uartcfg[1] != nil)
        uartinit(1, uartcfg[1]);

	/* Start reader threads AFTER viosharedinit and vgainit */
	proccreate(kbdreader, nil, 8192);
	proccreate(uartreader, (void*)0, 8192);
	proccreate(uartreader, (void*)1, 8192);

    for(i = 0; i < edevn; i++)
		if(edev[i](edevaux[i]) < 0)
        	sysfatal("%s: %r", edevt[i]);
    pcibusmap(); 

    mpmktable();
    acpimktables();

	pcidump();

	loadkernel(argv[0]);
  
	if(srvname != nil) init9p(srvname);
	if(kconfig != nil) kconfig();

    virtio_start_workers();  /* Start workers for CPU0 */

	vmx_cleanup_init();

	runloop();
	 
	exits(nil);
}